Skip to content

Commit 0719827

Browse files
Merge pull request #48 from scanoss/mscasso/feature/add-kb-config
Mscasso/feature/add db config
2 parents 6d75036 + 1083c85 commit 0719827

File tree

8 files changed

+139
-147
lines changed

8 files changed

+139
-147
lines changed

CHANGELOG.md

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
88
## [Unreleased]
99
### Added
1010
- Upcoming changes...
11+
## [1.4.4] - 2025-08-28
12+
### Added
13+
- Added option to select scanning KB.
14+
- `SCAN_KB_NAME` can be set to the scanning KB name. Default: oss.
15+
- Updated dependency versions
1116

1217
## [1.4.3] - 2025-02-04
1318
### Added

go.mod

Lines changed: 26 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -1,56 +1,55 @@
11
module scanoss.com/go-api
22

3-
go 1.22.0
3+
go 1.24.0
44

5-
toolchain go1.22.6
5+
toolchain go1.24.5
66

77
require (
88
github.com/go-co-op/gocron v1.37.0
99
github.com/golobby/config/v3 v3.4.2
1010
github.com/google/uuid v1.6.0
1111
github.com/gorilla/mux v1.8.1
1212
github.com/jpillora/ipfilter v1.2.9
13-
github.com/scanoss/zap-logging-helper v0.3.2
14-
github.com/stretchr/testify v1.10.0
15-
go.opentelemetry.io/contrib/instrumentation/github.com/gorilla/mux/otelmux v0.59.0
16-
go.opentelemetry.io/otel v1.34.0
17-
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.34.0
18-
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.34.0
19-
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.34.0
20-
go.opentelemetry.io/otel/metric v1.34.0
21-
go.opentelemetry.io/otel/sdk v1.34.0
22-
go.opentelemetry.io/otel/sdk/metric v1.34.0
23-
go.opentelemetry.io/otel/trace v1.34.0
13+
github.com/scanoss/zap-logging-helper v0.4.0
14+
github.com/stretchr/testify v1.11.1
15+
go.opentelemetry.io/contrib/instrumentation/github.com/gorilla/mux/otelmux v0.62.0
16+
go.opentelemetry.io/otel v1.37.0
17+
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.37.0
18+
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.37.0
19+
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.37.0
20+
go.opentelemetry.io/otel/metric v1.37.0
21+
go.opentelemetry.io/otel/sdk v1.37.0
22+
go.opentelemetry.io/otel/sdk/metric v1.37.0
23+
go.opentelemetry.io/otel/trace v1.37.0
2424
go.uber.org/zap v1.27.0
2525
)
2626

2727
require (
28-
github.com/BurntSushi/toml v1.3.2 // indirect
29-
github.com/cenkalti/backoff/v4 v4.3.0 // indirect
28+
github.com/BurntSushi/toml v1.5.0 // indirect
29+
github.com/cenkalti/backoff/v5 v5.0.3 // indirect
3030
github.com/davecgh/go-spew v1.1.1 // indirect
3131
github.com/felixge/httpsnoop v1.0.4 // indirect
32-
github.com/go-logr/logr v1.4.2 // indirect
32+
github.com/go-logr/logr v1.4.3 // indirect
3333
github.com/go-logr/stdr v1.2.2 // indirect
34-
github.com/golang/protobuf v1.5.4 // indirect
3534
github.com/golobby/cast v1.3.3 // indirect
3635
github.com/golobby/dotenv v1.3.2 // indirect
3736
github.com/golobby/env/v2 v2.2.4 // indirect
38-
github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.0 // indirect
39-
github.com/phuslu/iploc v1.0.20250131 // indirect
37+
github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.2 // indirect
38+
github.com/phuslu/iploc v1.0.20250815 // indirect
4039
github.com/pmezard/go-difflib v1.0.0 // indirect
4140
github.com/robfig/cron/v3 v3.0.1 // indirect
4241
github.com/tomasen/realip v0.0.0-20180522021738-f0c99a92ddce // indirect
4342
go.opentelemetry.io/auto/sdk v1.1.0 // indirect
44-
go.opentelemetry.io/proto/otlp v1.5.0 // indirect
43+
go.opentelemetry.io/proto/otlp v1.7.1 // indirect
4544
go.uber.org/atomic v1.11.0 // indirect
4645
go.uber.org/multierr v1.11.0 // indirect
47-
golang.org/x/net v0.34.0 // indirect
48-
golang.org/x/sys v0.29.0 // indirect
49-
golang.org/x/text v0.21.0 // indirect
50-
google.golang.org/genproto/googleapis/api v0.0.0-20250127172529-29210b9bc287 // indirect
51-
google.golang.org/genproto/googleapis/rpc v0.0.0-20250127172529-29210b9bc287 // indirect
52-
google.golang.org/grpc v1.70.0 // indirect
53-
google.golang.org/protobuf v1.36.4 // indirect
46+
golang.org/x/net v0.43.0 // indirect
47+
golang.org/x/sys v0.35.0 // indirect
48+
golang.org/x/text v0.28.0 // indirect
49+
google.golang.org/genproto/googleapis/api v0.0.0-20250826171959-ef028d996bc1 // indirect
50+
google.golang.org/genproto/googleapis/rpc v0.0.0-20250826171959-ef028d996bc1 // indirect
51+
google.golang.org/grpc v1.75.0 // indirect
52+
google.golang.org/protobuf v1.36.8 // indirect
5453
gopkg.in/yaml.v3 v3.0.1 // indirect
5554
)
5655

go.sum

Lines changed: 58 additions & 102 deletions
Large diffs are not rendered by default.

pkg/config/server_config.go

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,7 @@ type ServerConfig struct {
5757
Scanning struct {
5858
WfpLoc string `env:"SCAN_WFP_TMP"` // specific location to write temporary WFP files to
5959
ScanBinary string `env:"SCAN_BINARY"` // Binary to use for scanning
60+
ScanKbName string `env:"SCAN_KB_NAME"` // KB name passed as "-n" parameter to the scanoss command
6061
ScanDebug bool `env:"SCAN_DEBUG"` // true/false
6162
ScanFlags int `env:"SCAN_ENGINE_FLAGS"` // Default flags to use when scanning
6263
ScanTimeout int `env:"SCAN_ENGINE_TIMEOUT"` // timeout for waiting for the scan engine to respond
@@ -106,6 +107,7 @@ func setServerConfigDefaults(cfg *ServerConfig) {
106107
cfg.App.Mode = "dev"
107108
cfg.Logging.DynamicPort = "localhost:60085"
108109
cfg.Scanning.ScanBinary = "scanoss"
110+
cfg.Scanning.ScanKbName = "oss"
109111
cfg.Scanning.ScanFlags = 0
110112
cfg.Scanning.TmpFileDelete = true
111113
cfg.Scanning.Workers = 1 // Default to single threaded scanning

pkg/service/kb_details.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -85,7 +85,7 @@ func (s APIService) loadKBDetails() {
8585
engineVersion = "unknown"
8686
}
8787
// Load a random (hopefully non-existent) file match to extract the KB version details
88-
result, err := s.scanWfp("file=7c53a2de7dfeaa20d057db98468d6670,2321,path/to/dummy/file.txt", "", "", "", zs)
88+
result, err := s.scanWfp("file=7c53a2de7dfeaa20d057db98468d6670,2321,path/to/dummy/file.txt", "", "", "", "", zs)
8989
if err != nil {
9090
zs.Warnf("Failed to detect KB version from eninge: %v", err)
9191
return

pkg/service/scanning_service.go

Lines changed: 25 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -86,7 +86,7 @@ func (s APIService) scanDirect(w http.ResponseWriter, r *http.Request, zs *zap.S
8686
setSpanError(span, "No WFP contents supplied")
8787
return 0
8888
}
89-
flags, scanType, sbom := s.getFlags(r, zs)
89+
flags, scanType, sbom, dbName := s.getFlags(r, zs)
9090
// Check if we have an SBOM (and type) supplied
9191
var sbomFilename string
9292
if len(sbom) > 0 && len(scanType) > 0 {
@@ -121,9 +121,9 @@ func (s APIService) scanDirect(w http.ResponseWriter, r *http.Request, zs *zap.S
121121
s.countScanSize(wfps, wfpCount, zs, context, span)
122122
// Only one worker selected, so send the whole WFP in a single command
123123
if s.config.Scanning.Workers <= 1 {
124-
s.singleScan(string(contentsTrimmed), flags, scanType, sbomFilename, zs, w)
124+
s.singleScan(string(contentsTrimmed), flags, scanType, sbomFilename, dbName, zs, w)
125125
} else {
126-
s.scanThreaded(wfps, int(wfpCount), flags, scanType, sbomFilename, zs, w, span)
126+
s.scanThreaded(wfps, int(wfpCount), flags, scanType, sbomFilename, dbName, zs, w, span)
127127
}
128128
return wfpCount
129129
}
@@ -155,10 +155,11 @@ func (s APIService) countScanSize(wfps []string, wfpCount int64, zs *zap.Sugared
155155
}
156156

157157
// getFlags extracts the form values from a request returns the flags, scan type, and sbom data if detected.
158-
func (s APIService) getFlags(r *http.Request, zs *zap.SugaredLogger) (string, string, string) {
159-
flags := strings.TrimSpace(r.FormValue("flags")) // Check form for Scanning flags
160-
scanType := strings.TrimSpace(r.FormValue("type")) // Check form for SBOM type
161-
sbom := strings.TrimSpace(r.FormValue("assets")) // Check form for SBOM contents
158+
func (s APIService) getFlags(r *http.Request, zs *zap.SugaredLogger) (string, string, string, string) {
159+
flags := strings.TrimSpace(r.FormValue("flags")) // Check form for Scanning flags
160+
scanType := strings.TrimSpace(r.FormValue("type")) // Check form for SBOM type
161+
sbom := strings.TrimSpace(r.FormValue("assets")) // Check form for SBOM contents
162+
dbName := strings.TrimSpace(r.FormValue("db_name")) // Check form for db name
162163
// TODO is it necessary to check the header also for these values?
163164
if len(flags) == 0 {
164165
flags = strings.TrimSpace(r.Header.Get("flags")) // Check header for Scanning flags
@@ -169,10 +170,13 @@ func (s APIService) getFlags(r *http.Request, zs *zap.SugaredLogger) (string, st
169170
if len(sbom) == 0 {
170171
sbom = strings.TrimSpace(r.Header.Get("assets")) // Check header for SBOM contents
171172
}
173+
if len(dbName) == 0 {
174+
dbName = strings.TrimSpace(r.Header.Get("db_name")) // Check header for SBOM contents
175+
}
172176
if s.config.App.Trace {
173-
zs.Debugf("Header: %v, Form: %v, flags: %v, type: %v, assets: %v", r.Header, r.Form, flags, scanType, sbom)
177+
zs.Debugf("Header: %v, Form: %v, flags: %v, type: %v, assets: %v, db_name %v", r.Header, r.Form, flags, scanType, sbom, dbName)
174178
}
175-
return flags, scanType, sbom
179+
return flags, scanType, sbom, dbName
176180
}
177181

178182
// writeSbomFile writes the given string into an SBOM temporary file.
@@ -192,9 +196,9 @@ func (s APIService) writeSbomFile(sbom string, zs *zap.SugaredLogger) (*os.File,
192196
}
193197

194198
// singleScan runs a scan of the WFP in a single thread.
195-
func (s APIService) singleScan(wfp, flags, sbomType, sbomFile string, zs *zap.SugaredLogger, w http.ResponseWriter) {
199+
func (s APIService) singleScan(wfp, flags, sbomType, sbomFile, dbName string, zs *zap.SugaredLogger, w http.ResponseWriter) {
196200
zs.Debugf("Single threaded scan...")
197-
result, err := s.scanWfp(wfp, flags, sbomType, sbomFile, zs)
201+
result, err := s.scanWfp(wfp, flags, sbomType, sbomFile, dbName, zs)
198202
if err != nil {
199203
zs.Errorf("Engine scan failed: %v", err)
200204
http.Error(w, "ERROR engine scan failed", http.StatusInternalServerError)
@@ -212,7 +216,7 @@ func (s APIService) singleScan(wfp, flags, sbomType, sbomFile string, zs *zap.Su
212216
}
213217

214218
// scanThreaded scan the given WFPs in multiple threads.
215-
func (s APIService) scanThreaded(wfps []string, wfpCount int, flags, sbomType, sbomFile string, zs *zap.SugaredLogger, w http.ResponseWriter, span oteltrace.Span) {
219+
func (s APIService) scanThreaded(wfps []string, wfpCount int, flags, sbomType, sbomFile, dbName string, zs *zap.SugaredLogger, w http.ResponseWriter, span oteltrace.Span) {
216220
addSpanEvent(span, "Started Scanning.")
217221
numWorkers := s.config.Scanning.Workers
218222
groupedWfps := wfpCount / s.config.Scanning.WfpGrouping
@@ -229,7 +233,7 @@ func (s APIService) scanThreaded(wfps []string, wfpCount int, flags, sbomType, s
229233
zs.Debugf("Creating %v scanning workers...", numWorkers)
230234
// Create workers
231235
for i := 1; i <= numWorkers; i++ {
232-
go s.workerScan(fmt.Sprintf("%d_%s", i, uuid.New().String()), requests, results, flags, sbomType, sbomFile, zs)
236+
go s.workerScan(fmt.Sprintf("%d_%s", i, uuid.New().String()), requests, results, flags, sbomType, sbomFile, dbName, zs)
233237
}
234238
requestCount := 0 // Count the number of actual requests sent
235239
var wfpRequests []string
@@ -304,7 +308,7 @@ func (s APIService) validateHPSM(contents []byte, zs *zap.SugaredLogger, w http.
304308
}
305309

306310
// workerScan attempts to process all incoming scanning jobs and dumps the results into the subsequent results channel.
307-
func (s APIService) workerScan(id string, jobs <-chan string, results chan<- string, flags, sbomType, sbomFile string, zs *zap.SugaredLogger) {
311+
func (s APIService) workerScan(id string, jobs <-chan string, results chan<- string, flags, sbomType, sbomFile, dbName string, zs *zap.SugaredLogger) {
308312
if s.config.App.Trace {
309313
zs.Debugf("Starting up scanning worker: %v", id)
310314
}
@@ -318,7 +322,7 @@ func (s APIService) workerScan(id string, jobs <-chan string, results chan<- str
318322
zs.Warnf("Nothing in the job request to scan. Ignoring")
319323
results <- ""
320324
} else {
321-
result, err := s.scanWfp(job, flags, sbomType, sbomFile, zs)
325+
result, err := s.scanWfp(job, flags, sbomType, sbomFile, dbName, zs)
322326
if s.config.App.Trace {
323327
zs.Debugf("scan result (%v): %v, %v", id, result, err)
324328
}
@@ -343,7 +347,7 @@ func (s APIService) workerScan(id string, jobs <-chan string, results chan<- str
343347
}
344348

345349
// scanWfp run the scanoss engine scan of the supplied WFP.
346-
func (s APIService) scanWfp(wfp, flags, sbomType, sbomFile string, zs *zap.SugaredLogger) (string, error) {
350+
func (s APIService) scanWfp(wfp, flags, sbomType, sbomFile, dbName string, zs *zap.SugaredLogger) (string, error) {
347351
if len(wfp) == 0 {
348352
zs.Warnf("Nothing in the job request to scan. Ignoring")
349353
return "", fmt.Errorf("no wfp supplied to scan. ignoring")
@@ -367,6 +371,11 @@ func (s APIService) scanWfp(wfp, flags, sbomType, sbomFile string, zs *zap.Sugar
367371
if s.config.Scanning.ScanDebug {
368372
args = append(args, "-d") // Set debug mode
369373
}
374+
if len(dbName) > 0 && dbName != "" { // we want to prefer request over the local config
375+
args = append(args, fmt.Sprintf("-n%s", dbName))
376+
} else if s.config.Scanning.ScanKbName != "" { // Set scanning KB name
377+
args = append(args, fmt.Sprintf("-n%s", s.config.Scanning.ScanKbName))
378+
}
370379
if s.config.Scanning.ScanFlags > 0 { // Set system flags if enabled
371380
args = append(args, fmt.Sprintf("-F %v", s.config.Scanning.ScanFlags))
372381
} else if len(flags) > 0 && flags != "0" { // Set user supplied flags if enabled

test-support/scanoss.sh

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,19 @@ if [ "$1" == "-l" ] || [ "$2" == "-l" ] || [ "$3" == "-l" ] ; then
5151
exit 0
5252
fi
5353

54+
# Simulate invalid kb name
55+
for arg in "$@"; do
56+
if [[ "$arg" == "-n"* ]]; then
57+
# Extract everything after "-n"
58+
scf=${arg#-n}
59+
# Only show error if the value is NOT "oss"
60+
if [[ "$scf" != "oss" ]]; then
61+
echo "{Error: file and url tables must be present in $scf KB in order to proceed with the scan"
62+
exit 1
63+
fi
64+
fi
65+
done
66+
5467
# Simulate return a scan result
5568
if [ "$1" == "-w" ] || [ "$2" == "-w" ] || [ "$3" == "-w" ] || [ "$4" == "-w" ] || [ "$5" == "-w" ] || [ "$6" == "-w" ] || [ "$7" == "-w" ] || [ "$8" == "-w" ]; then
5669
for i in "$@"; do :; done

tests/scanning_test.go

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,10 +18,11 @@ package tests
1818

1919
import (
2020
"fmt"
21-
"github.com/stretchr/testify/suite"
2221
"io"
2322
"net/http"
2423
"testing"
24+
25+
"github.com/stretchr/testify/suite"
2526
)
2627

2728
type E2EScanningSuite struct {
@@ -42,6 +43,13 @@ func (s *E2EScanningSuite) TestScanning() {
4243
extraFields map[string]string
4344
want int
4445
}{
46+
{
47+
name: "Test Invalid KB name",
48+
filename: "../pkg/service/tests/fingers.wfp",
49+
shortName: "fingers.wfp",
50+
extraFields: map[string]string{"db_name": "test_kb"},
51+
want: http.StatusInternalServerError,
52+
},
4553
{
4654
name: "Test Empty WFP",
4755
filename: "../pkg/service/tests/fingers-empty.wfp",

0 commit comments

Comments
 (0)