From eb6a01bf7f799529508e87b3ba0c4046b99c0d40 Mon Sep 17 00:00:00 2001 From: Akkahshh Date: Tue, 13 May 2025 00:37:49 +0530 Subject: [PATCH 01/29] Add config model --- service/model/config.go | 66 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) create mode 100644 service/model/config.go diff --git a/service/model/config.go b/service/model/config.go new file mode 100644 index 0000000..b3cde36 --- /dev/null +++ b/service/model/config.go @@ -0,0 +1,66 @@ +package model + +type Config struct { + AppConfig AppConfig `mapstructure:"app"` + TemplateStorageConfig StorageConfig `mapstructure:"template_storage"` + FileStorageConfig StorageConfig `mapstructure:"file_storage"` + BrowserConfig BrowserConfig `mapstructure:"browser"` + WorkerPoolConfig WorkerPoolConfig `mapstructure:"workerpool"` + S3Config S3Config `mapstructure:"s3"` + AWSConfig AWSConfig `mapstructure:"aws"` + CertConfig map[string]CertConfig `mapstructure:"certificates"` + DBConfig DBConfig `mapstructure:"db"` +} + +type AppConfig struct { + LogLevel string `mapstructure:"log_level"` + EnableUI bool `mapstructure:"enable_ui"` + RodBrowserBin string `mapstructure:"rod_browser_bin"` +} + +type StorageConfig struct { + StorageType string `mapstructure:"storage_type"` +} + +type BrowserConfig struct { + TabPool int `mapstructure:"tab_pool"` +} + +type WorkerPoolConfig struct { + WorkerCount int `mapstructure:"worker_count"` + WorkerTimeoutMs int `mapstructure:"worker_timeout_ms"` +} + +type S3Config struct { + Endpoint string `mapstructure:"endpoint"` + Debug bool `mapstructure:"debug"` + Region string `mapstructure:"region"` + ForcePathStyle bool `mapstructure:"force_path_style"` + UploaderConcurrency int `mapstructure:"uploader_concurrency"` + UploaderPartSizeMB int64 `mapstructure:"uploader_part_size_mb"` + DownloaderConcurrency int `mapstructure:"downloader_concurrency"` + DownloaderPartSizeMB int64 `mapstructure:"downloader_part_size_mb"` + RetryMaxAttempts int `mapstructure:"retry_max_attempts"` + Bucket string `mapstructure:"bucket"` + UseCustomTransport bool `mapstructure:"use_custom_transport"` +} + +type AWSConfig struct { + AccessKeyID string `mapstructure:"access_key_id"` + SecretAccessKey string `mapstructure:"secret_access_key"` + SessionToken string `mapstructure:"session_token"` +} + +type CertConfig struct { + CertPath string `mapstructure:"cert_path"` + KeyPath string `mapstructure:"key_path"` + KeyPassword string `mapstructure:"key_password"` +} + +type DBConfig struct { + Host string `mapstructure:"host"` + Port int `mapstructure:"port"` + Username string `mapstructure:"username"` + Password string `mapstructure:"password"` + Database string `mapstructure:"database"` +} From 4f2f17f5c4ddf1ef68da0cc299bb18b48295e9d7 Mon Sep 17 00:00:00 2001 From: Akkahshh Date: Tue, 13 May 2025 00:38:35 +0530 Subject: [PATCH 02/29] Add config package to load config from file --- service/configs/config.go | 34 ++++++++++++++++++++++++++++++++++ service/main.go | 24 ++++++++++++++---------- 2 files changed, 48 insertions(+), 10 deletions(-) create mode 100644 service/configs/config.go diff --git a/service/configs/config.go b/service/configs/config.go new file mode 100644 index 0000000..1d04b73 --- /dev/null +++ b/service/configs/config.go @@ -0,0 +1,34 @@ +package config + +import ( + "github.com/Zomato/espresso/service/model" + "github.com/spf13/viper" +) + +func Load(filepath string) (model.Config, error) { + viper.SetConfigName("espressoconfig") // File name without extension + viper.SetConfigType("yaml") // File type + // Search paths relative to where the binary runs in container + viper.AddConfigPath(filepath) // Main config path in container + viper.AddConfigPath("../../configs") // For local development + viper.AddConfigPath("./configs") // Fallback path for local development + + viper.AutomaticEnv() + + err := viper.ReadInConfig() + if err != nil { + return model.Config{}, err + } + + var config model.Config + err = viper.Unmarshal(&config) + if err != nil { + return model.Config{}, err + } + + // TODO: Why are these fields set in the Dockerfile and not in the config.yaml file? + config.AppConfig.EnableUI = viper.GetBool("ENABLE_UI") + config.AppConfig.RodBrowserBin = viper.GetString("ROD_BROWSER_BIN") + + return config, nil +} diff --git a/service/main.go b/service/main.go index e3fd43a..0368ac0 100644 --- a/service/main.go +++ b/service/main.go @@ -11,30 +11,34 @@ import ( logger "github.com/Zomato/espresso/lib/logger" "github.com/Zomato/espresso/lib/workerpool" + config "github.com/Zomato/espresso/service/configs" "github.com/Zomato/espresso/service/controller/pdf_generation" - "github.com/Zomato/espresso/service/internal/pkg/viperpkg" "github.com/Zomato/espresso/service/utils" - "github.com/spf13/viper" ) func main() { ctx := context.Background() - viperpkg.InitConfig() + config, err := config.Load("/app/espresso/configs") + if err != nil { + log.Fatalf("Error loading config: %v", err) + } // Replace ZeroLog with any logging library by implementing ILogger interface. - zeroLog := utils.NewZeroLogger() + zeroLog := utils.NewZeroLogger(config.AppConfig.LogLevel) logger.Initialize(zeroLog) - log.Printf("Template storage type: %s", viper.GetString("template_storage.storage_type")) - log.Printf("File storage type: %s", viper.GetString("file_storage.storage_type")) + log.Printf("Template storage type: %s", config.TemplateStorageConfig.StorageType) + log.Printf("File storage type: %s", config.FileStorageConfig.StorageType) + + tabpool := config.BrowserConfig.TabPool - tabpool := viper.GetInt("browser.tab_pool") if err := browser_manager.Init(ctx, tabpool); err != nil { log.Fatalf("Failed to initialize browser: %v", err) } - workerCount := viper.GetInt("workerpool.worker_count") - workerTimeout := viper.GetInt("workerpool.worker_timeout") + + workerCount := config.WorkerPoolConfig.WorkerCount + workerTimeout := config.WorkerPoolConfig.WorkerTimeoutMs initializeWorkerPool(workerCount, workerTimeout) @@ -42,7 +46,7 @@ func main() { // Create a new ServeMux mux := http.NewServeMux() - pdf_generation.Register(mux) + pdf_generation.Register(mux, config) // Wrap the entire mux with the CORS middleware corsHandler := enableCORS(mux) From bd4716f26625bb8665544251c20312210bbab0e5 Mon Sep 17 00:00:00 2001 From: Akkahshh Date: Tue, 13 May 2025 00:39:08 +0530 Subject: [PATCH 03/29] Clean key names in yaml file --- service/configs/espressoconfig.yaml | 42 +++++++++++++++++------------ 1 file changed, 25 insertions(+), 17 deletions(-) diff --git a/service/configs/espressoconfig.yaml b/service/configs/espressoconfig.yaml index 192f849..94db74d 100644 --- a/service/configs/espressoconfig.yaml +++ b/service/configs/espressoconfig.yaml @@ -1,3 +1,6 @@ +app: + log_level: "debug" + template_storage: storage_type: "mysql" @@ -9,37 +12,42 @@ browser: workerpool: worker_count: 6 - worker_timeout: 310 # milliseconds + worker_timeout_ms: 310 # milliseconds s3: endpoint: "http://localstack:4566" debug: false region: "us-west-2" - forcePathStyle: true - uploaderConcurrency: 5 + force_path_style: true + uploader_concurrency: 5 # 5MB chunks - uploaderPartSize: 5 - downloaderConcurrency: 5 + uploader_part_size_mb: 5 + downloader_concurrency: 5 # 5MB chunks - downloaderPartSize: 5242880 - retryMaxAttempts: 3 + downloader_part_size_mb: 5242880 + retry_max_attempts: 3 bucket: "local-bucket" - useCustomTransport: false + use_custom_transport: false aws: - accessKeyID: "xxxxx-xxxxx-xxxxx-xxxxx-xxxxx" - secretAccessKey: "xxxxx-xxxxx-xxxxx-xxxxx-xxxxx" - sessionToken: "" + access_key_id: "xxxxx-xxxxx-xxxxx-xxxxx-xxxxx" + secret_access_key: "xxxxx-xxxxx-xxxxx-xxxxx-xxxxx" + session_token: "" -digital_certificates: +certificates: cert1: - cert_filepath: "./inputfiles/certificates/cert.pem" - key_filepath: "./inputfiles/certificates/key_pkcs8_encrypted.pem" + cert_path: "./inputfiles/certificates/cert.pem" + key_path: "./inputfiles/certificates/key_pkcs8_encrypted.pem" key_password: "test" cert2: - cert_filepath: "./certificates/certificate2.pem" - key_filepath: "./certificates/pirvatekey2.key" + cert_path: "./certificates/certificate2.pem" + key_path: "./certificates/pirvatekey2.key" key_password: "password2" -mysql: +db: + host: "mysql" + port: 3306 + username: "pdf_user" + password: "pdf_password" + database: "pdf_templates" dsn: "pdf_user:pdf_password@tcp(mysql:3306)/pdf_templates?parseTime=true" \ No newline at end of file From 74f40b9efae0fec4b661313f40aa3b58f1392078 Mon Sep 17 00:00:00 2001 From: Akkahshh Date: Tue, 13 May 2025 00:39:41 +0530 Subject: [PATCH 04/29] Set log level using config --- service/utils/zerolog.go | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/service/utils/zerolog.go b/service/utils/zerolog.go index 1dd4d1d..8c00f8a 100644 --- a/service/utils/zerolog.go +++ b/service/utils/zerolog.go @@ -14,9 +14,22 @@ type ZeroLog struct { logger zerolog.Logger } -func NewZeroLogger() ZeroLog { +func NewZeroLogger(level string) ZeroLog { log.Logger = log.Output(zerolog.ConsoleWriter{Out: os.Stderr, TimeFormat: time.RFC3339}) - zerolog.SetGlobalLevel(zerolog.DebugLevel) + // zerolog.SetGlobalLevel(zerolog.DebugLevel) + + switch level { + case "debug": + zerolog.SetGlobalLevel(zerolog.DebugLevel) + case "info": + zerolog.SetGlobalLevel(zerolog.InfoLevel) + case "warn": + zerolog.SetGlobalLevel(zerolog.WarnLevel) + case "error": + zerolog.SetGlobalLevel(zerolog.ErrorLevel) + default: + zerolog.SetGlobalLevel(zerolog.InfoLevel) + } return ZeroLog{ logger: log.Logger, From ce7e9dad56643e651780f967711ff537658b4833 Mon Sep 17 00:00:00 2001 From: Akkahshh Date: Tue, 13 May 2025 00:40:20 +0530 Subject: [PATCH 05/29] Pass config to espress service to create storage adapters --- service/controller/pdf_generation/register.go | 55 +++++++++++-------- 1 file changed, 32 insertions(+), 23 deletions(-) diff --git a/service/controller/pdf_generation/register.go b/service/controller/pdf_generation/register.go index e4e8454..96b4d4c 100644 --- a/service/controller/pdf_generation/register.go +++ b/service/controller/pdf_generation/register.go @@ -4,11 +4,10 @@ import ( "fmt" "log" "net/http" - "os" "github.com/Zomato/espresso/lib/s3" "github.com/Zomato/espresso/lib/templatestore" - "github.com/spf13/viper" + "github.com/Zomato/espresso/service/model" ) type EspressoService struct { @@ -16,41 +15,51 @@ type EspressoService struct { FileStorageAdapter *templatestore.StorageAdapter } -func NewEspressoService() (*EspressoService, error) { - templateStorageType := viper.GetString("template_storage.storage_type") - if os.Getenv("ENABLE_UI") == "true" && templateStorageType != templatestore.StorageAdapterTypeMySQL { +func NewEspressoService(config model.Config) (*EspressoService, error) { + templateStorageType := config.TemplateStorageConfig.StorageType + + if config.AppConfig.EnableUI && templateStorageType != templatestore.StorageAdapterTypeMySQL { return nil, fmt.Errorf("UI requires MySQL as template storage adapter, got: %s", templateStorageType) } + + mySqlDsn := fmt.Sprintf("%s:%s@tcp(%s:%d)/%s", + config.DBConfig.Username, + config.DBConfig.Password, + config.DBConfig.Host, + config.DBConfig.Port, + config.DBConfig.Database, + ) + templateStorageAdapter, err := templatestore.TemplateStorageAdapterFactory(&templatestore.StorageConfig{ StorageType: templateStorageType, // for s3 storage only S3Config: &s3.Config{ - Endpoint: viper.GetString("s3.endpoint"), - Region: viper.GetString("s3.region"), - Bucket: viper.GetString("s3.bucket"), - Debug: viper.GetBool("s3.debug"), - ForcePathStyle: viper.GetBool("s3.forcePathStyle"), - UploaderConcurrency: viper.GetInt("s3.uploaderConcurrency"), - UploaderPartSize: viper.GetInt64("s3.uploaderPartSize"), - DownloaderConcurrency: viper.GetInt("s3.downloaderConcurrency"), - DownloaderPartSize: viper.GetInt64("s3.downloaderPartSize"), - RetryMaxAttempts: viper.GetInt("s3.retryMaxAttempts"), - UseCustomTransport: viper.GetBool("s3.useCustomTransport"), + Endpoint: config.S3Config.Endpoint, + Region: config.S3Config.Region, + Bucket: config.S3Config.Bucket, + Debug: config.S3Config.Debug, + ForcePathStyle: config.S3Config.ForcePathStyle, + UploaderConcurrency: config.S3Config.UploaderConcurrency, + UploaderPartSize: config.S3Config.UploaderPartSizeMB, + DownloaderConcurrency: config.S3Config.DownloaderConcurrency, + DownloaderPartSize: config.S3Config.DownloaderPartSizeMB, + RetryMaxAttempts: config.S3Config.RetryMaxAttempts, + UseCustomTransport: config.S3Config.UseCustomTransport, }, // for s3 storage only AwsCredConfig: &s3.AwsCredConfig{ - AccessKeyID: viper.GetString("aws.accessKeyID"), - SecretAccessKey: viper.GetString("aws.secretAccessKey"), - SessionToken: viper.GetString("aws.sessionToken"), + AccessKeyID: config.AWSConfig.AccessKeyID, + SecretAccessKey: config.AWSConfig.SecretAccessKey, + SessionToken: config.AWSConfig.SessionToken, }, - MysqlDSN: viper.GetString("mysql.dsn"), // for mysql adapter + MysqlDSN: mySqlDsn, // for mysql adapter }) if err != nil { return nil, err } fileStorageAdapter, err := templatestore.TemplateStorageAdapterFactory(&templatestore.StorageConfig{ - StorageType: viper.GetString("file_storage.storage_type"), + StorageType: config.FileStorageConfig.StorageType, }) if err != nil { return nil, err @@ -58,8 +67,8 @@ func NewEspressoService() (*EspressoService, error) { return &EspressoService{TemplateStorageAdapter: &templateStorageAdapter, FileStorageAdapter: &fileStorageAdapter}, nil } -func Register(mux *http.ServeMux) { - espressoService, err := NewEspressoService() +func Register(mux *http.ServeMux, config model.Config) { + espressoService, err := NewEspressoService(config) if err != nil { log.Fatalf("Failed to initialize PDF service: %v", err) } From 1b52e61d048d903d744a3e4e549c916f89055422 Mon Sep 17 00:00:00 2001 From: Akkahshh Date: Tue, 13 May 2025 23:12:00 +0530 Subject: [PATCH 06/29] Move config package to service/internal/config --- service/{configs => internal/config}/config.go | 0 service/main.go | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) rename service/{configs => internal/config}/config.go (100%) diff --git a/service/configs/config.go b/service/internal/config/config.go similarity index 100% rename from service/configs/config.go rename to service/internal/config/config.go diff --git a/service/main.go b/service/main.go index 0368ac0..c331a2d 100644 --- a/service/main.go +++ b/service/main.go @@ -11,8 +11,8 @@ import ( logger "github.com/Zomato/espresso/lib/logger" "github.com/Zomato/espresso/lib/workerpool" - config "github.com/Zomato/espresso/service/configs" "github.com/Zomato/espresso/service/controller/pdf_generation" + "github.com/Zomato/espresso/service/internal/config" "github.com/Zomato/espresso/service/utils" ) From 11100ec7ecafaf3931ae7a48ac88131577f45ef4 Mon Sep 17 00:00:00 2001 From: Akkahshh Date: Wed, 14 May 2025 00:16:30 +0530 Subject: [PATCH 07/29] Add config constants --- service/internal/config/constant.go | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 service/internal/config/constant.go diff --git a/service/internal/config/constant.go b/service/internal/config/constant.go new file mode 100644 index 0000000..92dc8ca --- /dev/null +++ b/service/internal/config/constant.go @@ -0,0 +1,10 @@ +package config + +const ( + CONFIG_FILE_NAME = "CONFIG_FILE_NAME" + CONFIG_FILE_TYPE = "CONFIG_FILE_TYPE" + CONFIG_FILE_PATH = "CONFIG_FILE_PATH" + + ENABLE_UI = "ENABLE_UI" + ROD_BROWSER_BIN = "ROD_BROWSER_BIN" +) From 07ae542d7eeac7e886ed91d156cda8a7c27db29d Mon Sep 17 00:00:00 2001 From: Akkahshh Date: Wed, 14 May 2025 00:17:12 +0530 Subject: [PATCH 08/29] Take config options from env --- service/internal/config/config.go | 30 ++++++++++++++++++++---------- 1 file changed, 20 insertions(+), 10 deletions(-) diff --git a/service/internal/config/config.go b/service/internal/config/config.go index 1d04b73..a1b2781 100644 --- a/service/internal/config/config.go +++ b/service/internal/config/config.go @@ -1,20 +1,27 @@ package config import ( + "fmt" + "github.com/Zomato/espresso/service/model" "github.com/spf13/viper" ) -func Load(filepath string) (model.Config, error) { - viper.SetConfigName("espressoconfig") // File name without extension - viper.SetConfigType("yaml") // File type - // Search paths relative to where the binary runs in container - viper.AddConfigPath(filepath) // Main config path in container - viper.AddConfigPath("../../configs") // For local development - viper.AddConfigPath("./configs") // Fallback path for local development - +func Load() (model.Config, error) { viper.AutomaticEnv() + viper.SetDefault(CONFIG_FILE_NAME, "espressoconfig") + viper.SetDefault(CONFIG_FILE_TYPE, "yaml") + viper.SetDefault(CONFIG_FILE_PATH, "/app/espresso/configs") + + viper.SetConfigName(viper.GetString(CONFIG_FILE_NAME)) // File name without extension + viper.SetConfigType(viper.GetString(CONFIG_FILE_TYPE)) // File type + + // Search paths relative to where the binary runs in container + viper.AddConfigPath(CONFIG_FILE_PATH) // Main config path in container + viper.AddConfigPath("../../configs") // For local development + viper.AddConfigPath("./configs") // Fallback path for local development + err := viper.ReadInConfig() if err != nil { return model.Config{}, err @@ -27,8 +34,11 @@ func Load(filepath string) (model.Config, error) { } // TODO: Why are these fields set in the Dockerfile and not in the config.yaml file? - config.AppConfig.EnableUI = viper.GetBool("ENABLE_UI") - config.AppConfig.RodBrowserBin = viper.GetString("ROD_BROWSER_BIN") + config.AppConfig.EnableUI = viper.GetBool(ENABLE_UI) + config.AppConfig.RodBrowserBin = viper.GetString(ROD_BROWSER_BIN) + if config.AppConfig.RodBrowserBin == "" { + return model.Config{}, fmt.Errorf("environment variable %s not set", ROD_BROWSER_BIN) + } return config, nil } From f876badb5fff4ee8563ae76c95bef4bd08990e03 Mon Sep 17 00:00:00 2001 From: Akkahshh Date: Wed, 14 May 2025 00:18:00 +0530 Subject: [PATCH 09/29] Pass config to template storage adapter --- service/controller/pdf_generation/register.go | 29 ++++--------------- 1 file changed, 5 insertions(+), 24 deletions(-) diff --git a/service/controller/pdf_generation/register.go b/service/controller/pdf_generation/register.go index 96b4d4c..16192ad 100644 --- a/service/controller/pdf_generation/register.go +++ b/service/controller/pdf_generation/register.go @@ -5,7 +5,6 @@ import ( "log" "net/http" - "github.com/Zomato/espresso/lib/s3" "github.com/Zomato/espresso/lib/templatestore" "github.com/Zomato/espresso/service/model" ) @@ -22,7 +21,7 @@ func NewEspressoService(config model.Config) (*EspressoService, error) { return nil, fmt.Errorf("UI requires MySQL as template storage adapter, got: %s", templateStorageType) } - mySqlDsn := fmt.Sprintf("%s:%s@tcp(%s:%d)/%s", + mySqlDsn := fmt.Sprintf("%s:%s@tcp(%s:%d)/%s?parseTime=true", config.DBConfig.Username, config.DBConfig.Password, config.DBConfig.Host, @@ -31,28 +30,10 @@ func NewEspressoService(config model.Config) (*EspressoService, error) { ) templateStorageAdapter, err := templatestore.TemplateStorageAdapterFactory(&templatestore.StorageConfig{ - StorageType: templateStorageType, - // for s3 storage only - S3Config: &s3.Config{ - Endpoint: config.S3Config.Endpoint, - Region: config.S3Config.Region, - Bucket: config.S3Config.Bucket, - Debug: config.S3Config.Debug, - ForcePathStyle: config.S3Config.ForcePathStyle, - UploaderConcurrency: config.S3Config.UploaderConcurrency, - UploaderPartSize: config.S3Config.UploaderPartSizeMB, - DownloaderConcurrency: config.S3Config.DownloaderConcurrency, - DownloaderPartSize: config.S3Config.DownloaderPartSizeMB, - RetryMaxAttempts: config.S3Config.RetryMaxAttempts, - UseCustomTransport: config.S3Config.UseCustomTransport, - }, - // for s3 storage only - AwsCredConfig: &s3.AwsCredConfig{ - AccessKeyID: config.AWSConfig.AccessKeyID, - SecretAccessKey: config.AWSConfig.SecretAccessKey, - SessionToken: config.AWSConfig.SessionToken, - }, - MysqlDSN: mySqlDsn, // for mysql adapter + StorageType: templateStorageType, + S3Config: &config.S3Config, // for s3 storage only + AwsCredConfig: &config.AWSConfig, // for s3 storage only + MysqlDSN: mySqlDsn, // for mysql adapter }) if err != nil { return nil, err From c435d1746e401e0649f95267a70c7a8d922305ad Mon Sep 17 00:00:00 2001 From: Akkahshh Date: Wed, 14 May 2025 00:18:34 +0530 Subject: [PATCH 10/29] Add few keys in config file --- service/configs/espressoconfig.yaml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/service/configs/espressoconfig.yaml b/service/configs/espressoconfig.yaml index 94db74d..beb81d2 100644 --- a/service/configs/espressoconfig.yaml +++ b/service/configs/espressoconfig.yaml @@ -1,5 +1,8 @@ app: log_level: "debug" + server_port: 8081 + enable_ui: true + rod_browser_bin: "/opt/homebrew/bin/chromium" template_storage: storage_type: "mysql" @@ -45,9 +48,8 @@ certificates: key_password: "password2" db: - host: "mysql" - port: 3306 + host: "localhost" + port: 3308 username: "pdf_user" password: "pdf_password" - database: "pdf_templates" - dsn: "pdf_user:pdf_password@tcp(mysql:3306)/pdf_templates?parseTime=true" \ No newline at end of file + database: "pdf_templates" \ No newline at end of file From 4182e1354c9ea64f331c1384b68e6dc15bcdc47a Mon Sep 17 00:00:00 2001 From: Akkahshh Date: Wed, 14 May 2025 00:19:11 +0530 Subject: [PATCH 11/29] Change config model --- service/main.go | 10 ++++++---- service/model/config.go | 29 +++++++---------------------- 2 files changed, 13 insertions(+), 26 deletions(-) diff --git a/service/main.go b/service/main.go index c331a2d..a6731ce 100644 --- a/service/main.go +++ b/service/main.go @@ -19,7 +19,7 @@ import ( func main() { ctx := context.Background() - config, err := config.Load("/app/espresso/configs") + config, err := config.Load() if err != nil { log.Fatalf("Error loading config: %v", err) } @@ -32,8 +32,9 @@ func main() { log.Printf("File storage type: %s", config.FileStorageConfig.StorageType) tabpool := config.BrowserConfig.TabPool + browserPath := config.AppConfig.RodBrowserBin - if err := browser_manager.Init(ctx, tabpool); err != nil { + if err := browser_manager.Init(ctx, tabpool, browserPath); err != nil { log.Fatalf("Failed to initialize browser: %v", err) } @@ -50,8 +51,9 @@ func main() { // Wrap the entire mux with the CORS middleware corsHandler := enableCORS(mux) - log.Println("Starting PDF client server on :8081") - if err := http.ListenAndServe(":8081", corsHandler); err != nil { + port := fmt.Sprintf(":%d", config.AppConfig.ServerPort) + log.Printf("Starting PDF client server on: %s\n", port) + if err := http.ListenAndServe(port, corsHandler); err != nil { log.Fatal(err) } diff --git a/service/model/config.go b/service/model/config.go index b3cde36..418bea6 100644 --- a/service/model/config.go +++ b/service/model/config.go @@ -1,19 +1,24 @@ package model +import ( + "github.com/Zomato/espresso/lib/s3" +) + type Config struct { AppConfig AppConfig `mapstructure:"app"` TemplateStorageConfig StorageConfig `mapstructure:"template_storage"` FileStorageConfig StorageConfig `mapstructure:"file_storage"` BrowserConfig BrowserConfig `mapstructure:"browser"` WorkerPoolConfig WorkerPoolConfig `mapstructure:"workerpool"` - S3Config S3Config `mapstructure:"s3"` - AWSConfig AWSConfig `mapstructure:"aws"` + S3Config s3.Config `mapstructure:"s3"` + AWSConfig s3.AwsCredConfig `mapstructure:"aws"` CertConfig map[string]CertConfig `mapstructure:"certificates"` DBConfig DBConfig `mapstructure:"db"` } type AppConfig struct { LogLevel string `mapstructure:"log_level"` + ServerPort int `mapstructure:"server_port"` EnableUI bool `mapstructure:"enable_ui"` RodBrowserBin string `mapstructure:"rod_browser_bin"` } @@ -31,26 +36,6 @@ type WorkerPoolConfig struct { WorkerTimeoutMs int `mapstructure:"worker_timeout_ms"` } -type S3Config struct { - Endpoint string `mapstructure:"endpoint"` - Debug bool `mapstructure:"debug"` - Region string `mapstructure:"region"` - ForcePathStyle bool `mapstructure:"force_path_style"` - UploaderConcurrency int `mapstructure:"uploader_concurrency"` - UploaderPartSizeMB int64 `mapstructure:"uploader_part_size_mb"` - DownloaderConcurrency int `mapstructure:"downloader_concurrency"` - DownloaderPartSizeMB int64 `mapstructure:"downloader_part_size_mb"` - RetryMaxAttempts int `mapstructure:"retry_max_attempts"` - Bucket string `mapstructure:"bucket"` - UseCustomTransport bool `mapstructure:"use_custom_transport"` -} - -type AWSConfig struct { - AccessKeyID string `mapstructure:"access_key_id"` - SecretAccessKey string `mapstructure:"secret_access_key"` - SessionToken string `mapstructure:"session_token"` -} - type CertConfig struct { CertPath string `mapstructure:"cert_path"` KeyPath string `mapstructure:"key_path"` From fdb494ec766b53f61aeef8020b2de776ffe7912d Mon Sep 17 00:00:00 2001 From: Akkahshh Date: Tue, 13 May 2025 00:37:49 +0530 Subject: [PATCH 12/29] Add config model --- service/model/config.go | 66 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) create mode 100644 service/model/config.go diff --git a/service/model/config.go b/service/model/config.go new file mode 100644 index 0000000..b3cde36 --- /dev/null +++ b/service/model/config.go @@ -0,0 +1,66 @@ +package model + +type Config struct { + AppConfig AppConfig `mapstructure:"app"` + TemplateStorageConfig StorageConfig `mapstructure:"template_storage"` + FileStorageConfig StorageConfig `mapstructure:"file_storage"` + BrowserConfig BrowserConfig `mapstructure:"browser"` + WorkerPoolConfig WorkerPoolConfig `mapstructure:"workerpool"` + S3Config S3Config `mapstructure:"s3"` + AWSConfig AWSConfig `mapstructure:"aws"` + CertConfig map[string]CertConfig `mapstructure:"certificates"` + DBConfig DBConfig `mapstructure:"db"` +} + +type AppConfig struct { + LogLevel string `mapstructure:"log_level"` + EnableUI bool `mapstructure:"enable_ui"` + RodBrowserBin string `mapstructure:"rod_browser_bin"` +} + +type StorageConfig struct { + StorageType string `mapstructure:"storage_type"` +} + +type BrowserConfig struct { + TabPool int `mapstructure:"tab_pool"` +} + +type WorkerPoolConfig struct { + WorkerCount int `mapstructure:"worker_count"` + WorkerTimeoutMs int `mapstructure:"worker_timeout_ms"` +} + +type S3Config struct { + Endpoint string `mapstructure:"endpoint"` + Debug bool `mapstructure:"debug"` + Region string `mapstructure:"region"` + ForcePathStyle bool `mapstructure:"force_path_style"` + UploaderConcurrency int `mapstructure:"uploader_concurrency"` + UploaderPartSizeMB int64 `mapstructure:"uploader_part_size_mb"` + DownloaderConcurrency int `mapstructure:"downloader_concurrency"` + DownloaderPartSizeMB int64 `mapstructure:"downloader_part_size_mb"` + RetryMaxAttempts int `mapstructure:"retry_max_attempts"` + Bucket string `mapstructure:"bucket"` + UseCustomTransport bool `mapstructure:"use_custom_transport"` +} + +type AWSConfig struct { + AccessKeyID string `mapstructure:"access_key_id"` + SecretAccessKey string `mapstructure:"secret_access_key"` + SessionToken string `mapstructure:"session_token"` +} + +type CertConfig struct { + CertPath string `mapstructure:"cert_path"` + KeyPath string `mapstructure:"key_path"` + KeyPassword string `mapstructure:"key_password"` +} + +type DBConfig struct { + Host string `mapstructure:"host"` + Port int `mapstructure:"port"` + Username string `mapstructure:"username"` + Password string `mapstructure:"password"` + Database string `mapstructure:"database"` +} From 0ea8db38481d5cf529b54021e4ac363f1e82ef96 Mon Sep 17 00:00:00 2001 From: Akkahshh Date: Tue, 13 May 2025 00:38:35 +0530 Subject: [PATCH 13/29] Add config package to load config from file --- service/configs/config.go | 34 ++++++++++++++++++++++++++++++++++ service/main.go | 25 +++++++++++++------------ 2 files changed, 47 insertions(+), 12 deletions(-) create mode 100644 service/configs/config.go diff --git a/service/configs/config.go b/service/configs/config.go new file mode 100644 index 0000000..1d04b73 --- /dev/null +++ b/service/configs/config.go @@ -0,0 +1,34 @@ +package config + +import ( + "github.com/Zomato/espresso/service/model" + "github.com/spf13/viper" +) + +func Load(filepath string) (model.Config, error) { + viper.SetConfigName("espressoconfig") // File name without extension + viper.SetConfigType("yaml") // File type + // Search paths relative to where the binary runs in container + viper.AddConfigPath(filepath) // Main config path in container + viper.AddConfigPath("../../configs") // For local development + viper.AddConfigPath("./configs") // Fallback path for local development + + viper.AutomaticEnv() + + err := viper.ReadInConfig() + if err != nil { + return model.Config{}, err + } + + var config model.Config + err = viper.Unmarshal(&config) + if err != nil { + return model.Config{}, err + } + + // TODO: Why are these fields set in the Dockerfile and not in the config.yaml file? + config.AppConfig.EnableUI = viper.GetBool("ENABLE_UI") + config.AppConfig.RodBrowserBin = viper.GetString("ROD_BROWSER_BIN") + + return config, nil +} diff --git a/service/main.go b/service/main.go index 3cebab0..f4305e8 100644 --- a/service/main.go +++ b/service/main.go @@ -10,33 +10,34 @@ import ( logger "github.com/Zomato/espresso/lib/logger" "github.com/Zomato/espresso/lib/workerpool" + config "github.com/Zomato/espresso/service/configs" "github.com/Zomato/espresso/service/controller/pdf_generation" - "github.com/Zomato/espresso/service/internal/pkg/viperpkg" "github.com/Zomato/espresso/service/utils" - "github.com/spf13/viper" ) func main() { ctx := context.Background() - viperpkg.InitConfig() + config, err := config.Load("/app/espresso/configs") + if err != nil { + log.Fatalf("Error loading config: %v", err) + } // Replace ZeroLog with any logging library by implementing ILogger interface. - zeroLog := utils.NewZeroLogger() + zeroLog := utils.NewZeroLogger(config.AppConfig.LogLevel) logger.Initialize(zeroLog) - templateStorageType := viper.GetString("template_storage.storage_type") - zeroLog.Info(ctx, "Template storage type ", map[string]any{"type": templateStorageType}) + log.Printf("Template storage type: %s", config.TemplateStorageConfig.StorageType) + log.Printf("File storage type: %s", config.FileStorageConfig.StorageType) - fileStorageType := viper.GetString("file_storage.storage_type") - zeroLog.Info(ctx, "File storage type ", map[string]any{"type": fileStorageType}) + tabpool := config.BrowserConfig.TabPool - tabpool := viper.GetInt("browser.tab_pool") if err := browser_manager.Init(ctx, tabpool); err != nil { log.Fatalf("Failed to initialize browser: %v", err) } - workerCount := viper.GetInt("workerpool.worker_count") - workerTimeout := viper.GetInt("workerpool.worker_timeout") + + workerCount := config.WorkerPoolConfig.WorkerCount + workerTimeout := config.WorkerPoolConfig.WorkerTimeoutMs initializeWorkerPool(workerCount, workerTimeout) @@ -44,7 +45,7 @@ func main() { // Create a new ServeMux mux := http.NewServeMux() - pdf_generation.Register(mux) + pdf_generation.Register(mux, config) // Wrap the entire mux with the CORS middleware corsHandler := enableCORS(mux) From 80e754edeb95f27de049b23a27826754b31439ef Mon Sep 17 00:00:00 2001 From: Akkahshh Date: Tue, 13 May 2025 00:39:08 +0530 Subject: [PATCH 14/29] Clean key names in yaml file --- service/configs/espressoconfig.yaml | 42 +++++++++++++++++------------ 1 file changed, 25 insertions(+), 17 deletions(-) diff --git a/service/configs/espressoconfig.yaml b/service/configs/espressoconfig.yaml index 192f849..94db74d 100644 --- a/service/configs/espressoconfig.yaml +++ b/service/configs/espressoconfig.yaml @@ -1,3 +1,6 @@ +app: + log_level: "debug" + template_storage: storage_type: "mysql" @@ -9,37 +12,42 @@ browser: workerpool: worker_count: 6 - worker_timeout: 310 # milliseconds + worker_timeout_ms: 310 # milliseconds s3: endpoint: "http://localstack:4566" debug: false region: "us-west-2" - forcePathStyle: true - uploaderConcurrency: 5 + force_path_style: true + uploader_concurrency: 5 # 5MB chunks - uploaderPartSize: 5 - downloaderConcurrency: 5 + uploader_part_size_mb: 5 + downloader_concurrency: 5 # 5MB chunks - downloaderPartSize: 5242880 - retryMaxAttempts: 3 + downloader_part_size_mb: 5242880 + retry_max_attempts: 3 bucket: "local-bucket" - useCustomTransport: false + use_custom_transport: false aws: - accessKeyID: "xxxxx-xxxxx-xxxxx-xxxxx-xxxxx" - secretAccessKey: "xxxxx-xxxxx-xxxxx-xxxxx-xxxxx" - sessionToken: "" + access_key_id: "xxxxx-xxxxx-xxxxx-xxxxx-xxxxx" + secret_access_key: "xxxxx-xxxxx-xxxxx-xxxxx-xxxxx" + session_token: "" -digital_certificates: +certificates: cert1: - cert_filepath: "./inputfiles/certificates/cert.pem" - key_filepath: "./inputfiles/certificates/key_pkcs8_encrypted.pem" + cert_path: "./inputfiles/certificates/cert.pem" + key_path: "./inputfiles/certificates/key_pkcs8_encrypted.pem" key_password: "test" cert2: - cert_filepath: "./certificates/certificate2.pem" - key_filepath: "./certificates/pirvatekey2.key" + cert_path: "./certificates/certificate2.pem" + key_path: "./certificates/pirvatekey2.key" key_password: "password2" -mysql: +db: + host: "mysql" + port: 3306 + username: "pdf_user" + password: "pdf_password" + database: "pdf_templates" dsn: "pdf_user:pdf_password@tcp(mysql:3306)/pdf_templates?parseTime=true" \ No newline at end of file From f3e7e754fdb714f79fea60bfc23e96547dc8d614 Mon Sep 17 00:00:00 2001 From: Akkahshh Date: Tue, 13 May 2025 00:39:41 +0530 Subject: [PATCH 15/29] Set log level using config --- service/utils/zerolog.go | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/service/utils/zerolog.go b/service/utils/zerolog.go index ef5ed9e..fb217c8 100644 --- a/service/utils/zerolog.go +++ b/service/utils/zerolog.go @@ -18,13 +18,22 @@ var ( Logger ZeroLog ) -func NewZeroLogger() ZeroLog { - log.Logger = log.Output(zerolog.ConsoleWriter{ - Out: os.Stderr, - TimeFormat: time.RFC3339, - }) +func NewZeroLogger(level string) ZeroLog { + log.Logger = log.Output(zerolog.ConsoleWriter{Out: os.Stderr, TimeFormat: time.RFC3339}) + // zerolog.SetGlobalLevel(zerolog.DebugLevel) - zerolog.SetGlobalLevel(zerolog.DebugLevel) + switch level { + case "debug": + zerolog.SetGlobalLevel(zerolog.DebugLevel) + case "info": + zerolog.SetGlobalLevel(zerolog.InfoLevel) + case "warn": + zerolog.SetGlobalLevel(zerolog.WarnLevel) + case "error": + zerolog.SetGlobalLevel(zerolog.ErrorLevel) + default: + zerolog.SetGlobalLevel(zerolog.InfoLevel) + } zeroLog := ZeroLog{ logger: log.Logger, From 8ad2a6b659cf611005cb6dc386accf5774b934dd Mon Sep 17 00:00:00 2001 From: Akkahshh Date: Tue, 13 May 2025 00:40:20 +0530 Subject: [PATCH 16/29] Pass config to espress service to create storage adapters --- service/controller/pdf_generation/register.go | 55 +++++++++++-------- 1 file changed, 32 insertions(+), 23 deletions(-) diff --git a/service/controller/pdf_generation/register.go b/service/controller/pdf_generation/register.go index e4e8454..96b4d4c 100644 --- a/service/controller/pdf_generation/register.go +++ b/service/controller/pdf_generation/register.go @@ -4,11 +4,10 @@ import ( "fmt" "log" "net/http" - "os" "github.com/Zomato/espresso/lib/s3" "github.com/Zomato/espresso/lib/templatestore" - "github.com/spf13/viper" + "github.com/Zomato/espresso/service/model" ) type EspressoService struct { @@ -16,41 +15,51 @@ type EspressoService struct { FileStorageAdapter *templatestore.StorageAdapter } -func NewEspressoService() (*EspressoService, error) { - templateStorageType := viper.GetString("template_storage.storage_type") - if os.Getenv("ENABLE_UI") == "true" && templateStorageType != templatestore.StorageAdapterTypeMySQL { +func NewEspressoService(config model.Config) (*EspressoService, error) { + templateStorageType := config.TemplateStorageConfig.StorageType + + if config.AppConfig.EnableUI && templateStorageType != templatestore.StorageAdapterTypeMySQL { return nil, fmt.Errorf("UI requires MySQL as template storage adapter, got: %s", templateStorageType) } + + mySqlDsn := fmt.Sprintf("%s:%s@tcp(%s:%d)/%s", + config.DBConfig.Username, + config.DBConfig.Password, + config.DBConfig.Host, + config.DBConfig.Port, + config.DBConfig.Database, + ) + templateStorageAdapter, err := templatestore.TemplateStorageAdapterFactory(&templatestore.StorageConfig{ StorageType: templateStorageType, // for s3 storage only S3Config: &s3.Config{ - Endpoint: viper.GetString("s3.endpoint"), - Region: viper.GetString("s3.region"), - Bucket: viper.GetString("s3.bucket"), - Debug: viper.GetBool("s3.debug"), - ForcePathStyle: viper.GetBool("s3.forcePathStyle"), - UploaderConcurrency: viper.GetInt("s3.uploaderConcurrency"), - UploaderPartSize: viper.GetInt64("s3.uploaderPartSize"), - DownloaderConcurrency: viper.GetInt("s3.downloaderConcurrency"), - DownloaderPartSize: viper.GetInt64("s3.downloaderPartSize"), - RetryMaxAttempts: viper.GetInt("s3.retryMaxAttempts"), - UseCustomTransport: viper.GetBool("s3.useCustomTransport"), + Endpoint: config.S3Config.Endpoint, + Region: config.S3Config.Region, + Bucket: config.S3Config.Bucket, + Debug: config.S3Config.Debug, + ForcePathStyle: config.S3Config.ForcePathStyle, + UploaderConcurrency: config.S3Config.UploaderConcurrency, + UploaderPartSize: config.S3Config.UploaderPartSizeMB, + DownloaderConcurrency: config.S3Config.DownloaderConcurrency, + DownloaderPartSize: config.S3Config.DownloaderPartSizeMB, + RetryMaxAttempts: config.S3Config.RetryMaxAttempts, + UseCustomTransport: config.S3Config.UseCustomTransport, }, // for s3 storage only AwsCredConfig: &s3.AwsCredConfig{ - AccessKeyID: viper.GetString("aws.accessKeyID"), - SecretAccessKey: viper.GetString("aws.secretAccessKey"), - SessionToken: viper.GetString("aws.sessionToken"), + AccessKeyID: config.AWSConfig.AccessKeyID, + SecretAccessKey: config.AWSConfig.SecretAccessKey, + SessionToken: config.AWSConfig.SessionToken, }, - MysqlDSN: viper.GetString("mysql.dsn"), // for mysql adapter + MysqlDSN: mySqlDsn, // for mysql adapter }) if err != nil { return nil, err } fileStorageAdapter, err := templatestore.TemplateStorageAdapterFactory(&templatestore.StorageConfig{ - StorageType: viper.GetString("file_storage.storage_type"), + StorageType: config.FileStorageConfig.StorageType, }) if err != nil { return nil, err @@ -58,8 +67,8 @@ func NewEspressoService() (*EspressoService, error) { return &EspressoService{TemplateStorageAdapter: &templateStorageAdapter, FileStorageAdapter: &fileStorageAdapter}, nil } -func Register(mux *http.ServeMux) { - espressoService, err := NewEspressoService() +func Register(mux *http.ServeMux, config model.Config) { + espressoService, err := NewEspressoService(config) if err != nil { log.Fatalf("Failed to initialize PDF service: %v", err) } From d876833d907d5a8c79292e9d704e0a3e84699b8f Mon Sep 17 00:00:00 2001 From: Akkahshh Date: Tue, 13 May 2025 23:12:00 +0530 Subject: [PATCH 17/29] Move config package to service/internal/config --- service/{configs => internal/config}/config.go | 0 service/main.go | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) rename service/{configs => internal/config}/config.go (100%) diff --git a/service/configs/config.go b/service/internal/config/config.go similarity index 100% rename from service/configs/config.go rename to service/internal/config/config.go diff --git a/service/main.go b/service/main.go index f4305e8..affe4a5 100644 --- a/service/main.go +++ b/service/main.go @@ -10,8 +10,8 @@ import ( logger "github.com/Zomato/espresso/lib/logger" "github.com/Zomato/espresso/lib/workerpool" - config "github.com/Zomato/espresso/service/configs" "github.com/Zomato/espresso/service/controller/pdf_generation" + "github.com/Zomato/espresso/service/internal/config" "github.com/Zomato/espresso/service/utils" ) From 5e79330443d4ba3b1f3d61a97c22fbeb5d62e56e Mon Sep 17 00:00:00 2001 From: Akkahshh Date: Wed, 14 May 2025 00:16:30 +0530 Subject: [PATCH 18/29] Add config constants --- service/internal/config/constant.go | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 service/internal/config/constant.go diff --git a/service/internal/config/constant.go b/service/internal/config/constant.go new file mode 100644 index 0000000..92dc8ca --- /dev/null +++ b/service/internal/config/constant.go @@ -0,0 +1,10 @@ +package config + +const ( + CONFIG_FILE_NAME = "CONFIG_FILE_NAME" + CONFIG_FILE_TYPE = "CONFIG_FILE_TYPE" + CONFIG_FILE_PATH = "CONFIG_FILE_PATH" + + ENABLE_UI = "ENABLE_UI" + ROD_BROWSER_BIN = "ROD_BROWSER_BIN" +) From 51eae0a417f5c429f217847acf2aec6fc473889a Mon Sep 17 00:00:00 2001 From: Akkahshh Date: Wed, 14 May 2025 00:17:12 +0530 Subject: [PATCH 19/29] Take config options from env --- service/internal/config/config.go | 30 ++++++++++++++++++++---------- 1 file changed, 20 insertions(+), 10 deletions(-) diff --git a/service/internal/config/config.go b/service/internal/config/config.go index 1d04b73..a1b2781 100644 --- a/service/internal/config/config.go +++ b/service/internal/config/config.go @@ -1,20 +1,27 @@ package config import ( + "fmt" + "github.com/Zomato/espresso/service/model" "github.com/spf13/viper" ) -func Load(filepath string) (model.Config, error) { - viper.SetConfigName("espressoconfig") // File name without extension - viper.SetConfigType("yaml") // File type - // Search paths relative to where the binary runs in container - viper.AddConfigPath(filepath) // Main config path in container - viper.AddConfigPath("../../configs") // For local development - viper.AddConfigPath("./configs") // Fallback path for local development - +func Load() (model.Config, error) { viper.AutomaticEnv() + viper.SetDefault(CONFIG_FILE_NAME, "espressoconfig") + viper.SetDefault(CONFIG_FILE_TYPE, "yaml") + viper.SetDefault(CONFIG_FILE_PATH, "/app/espresso/configs") + + viper.SetConfigName(viper.GetString(CONFIG_FILE_NAME)) // File name without extension + viper.SetConfigType(viper.GetString(CONFIG_FILE_TYPE)) // File type + + // Search paths relative to where the binary runs in container + viper.AddConfigPath(CONFIG_FILE_PATH) // Main config path in container + viper.AddConfigPath("../../configs") // For local development + viper.AddConfigPath("./configs") // Fallback path for local development + err := viper.ReadInConfig() if err != nil { return model.Config{}, err @@ -27,8 +34,11 @@ func Load(filepath string) (model.Config, error) { } // TODO: Why are these fields set in the Dockerfile and not in the config.yaml file? - config.AppConfig.EnableUI = viper.GetBool("ENABLE_UI") - config.AppConfig.RodBrowserBin = viper.GetString("ROD_BROWSER_BIN") + config.AppConfig.EnableUI = viper.GetBool(ENABLE_UI) + config.AppConfig.RodBrowserBin = viper.GetString(ROD_BROWSER_BIN) + if config.AppConfig.RodBrowserBin == "" { + return model.Config{}, fmt.Errorf("environment variable %s not set", ROD_BROWSER_BIN) + } return config, nil } From 051b9efff9deb5ee3e869ae32d5a09b7959a2093 Mon Sep 17 00:00:00 2001 From: Akkahshh Date: Wed, 14 May 2025 00:18:00 +0530 Subject: [PATCH 20/29] Pass config to template storage adapter --- service/controller/pdf_generation/register.go | 29 ++++--------------- 1 file changed, 5 insertions(+), 24 deletions(-) diff --git a/service/controller/pdf_generation/register.go b/service/controller/pdf_generation/register.go index 96b4d4c..16192ad 100644 --- a/service/controller/pdf_generation/register.go +++ b/service/controller/pdf_generation/register.go @@ -5,7 +5,6 @@ import ( "log" "net/http" - "github.com/Zomato/espresso/lib/s3" "github.com/Zomato/espresso/lib/templatestore" "github.com/Zomato/espresso/service/model" ) @@ -22,7 +21,7 @@ func NewEspressoService(config model.Config) (*EspressoService, error) { return nil, fmt.Errorf("UI requires MySQL as template storage adapter, got: %s", templateStorageType) } - mySqlDsn := fmt.Sprintf("%s:%s@tcp(%s:%d)/%s", + mySqlDsn := fmt.Sprintf("%s:%s@tcp(%s:%d)/%s?parseTime=true", config.DBConfig.Username, config.DBConfig.Password, config.DBConfig.Host, @@ -31,28 +30,10 @@ func NewEspressoService(config model.Config) (*EspressoService, error) { ) templateStorageAdapter, err := templatestore.TemplateStorageAdapterFactory(&templatestore.StorageConfig{ - StorageType: templateStorageType, - // for s3 storage only - S3Config: &s3.Config{ - Endpoint: config.S3Config.Endpoint, - Region: config.S3Config.Region, - Bucket: config.S3Config.Bucket, - Debug: config.S3Config.Debug, - ForcePathStyle: config.S3Config.ForcePathStyle, - UploaderConcurrency: config.S3Config.UploaderConcurrency, - UploaderPartSize: config.S3Config.UploaderPartSizeMB, - DownloaderConcurrency: config.S3Config.DownloaderConcurrency, - DownloaderPartSize: config.S3Config.DownloaderPartSizeMB, - RetryMaxAttempts: config.S3Config.RetryMaxAttempts, - UseCustomTransport: config.S3Config.UseCustomTransport, - }, - // for s3 storage only - AwsCredConfig: &s3.AwsCredConfig{ - AccessKeyID: config.AWSConfig.AccessKeyID, - SecretAccessKey: config.AWSConfig.SecretAccessKey, - SessionToken: config.AWSConfig.SessionToken, - }, - MysqlDSN: mySqlDsn, // for mysql adapter + StorageType: templateStorageType, + S3Config: &config.S3Config, // for s3 storage only + AwsCredConfig: &config.AWSConfig, // for s3 storage only + MysqlDSN: mySqlDsn, // for mysql adapter }) if err != nil { return nil, err From 6161ada464e3c26e151513f2717e3174edf4e12f Mon Sep 17 00:00:00 2001 From: Akkahshh Date: Wed, 14 May 2025 00:18:34 +0530 Subject: [PATCH 21/29] Add few keys in config file --- service/configs/espressoconfig.yaml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/service/configs/espressoconfig.yaml b/service/configs/espressoconfig.yaml index 94db74d..beb81d2 100644 --- a/service/configs/espressoconfig.yaml +++ b/service/configs/espressoconfig.yaml @@ -1,5 +1,8 @@ app: log_level: "debug" + server_port: 8081 + enable_ui: true + rod_browser_bin: "/opt/homebrew/bin/chromium" template_storage: storage_type: "mysql" @@ -45,9 +48,8 @@ certificates: key_password: "password2" db: - host: "mysql" - port: 3306 + host: "localhost" + port: 3308 username: "pdf_user" password: "pdf_password" - database: "pdf_templates" - dsn: "pdf_user:pdf_password@tcp(mysql:3306)/pdf_templates?parseTime=true" \ No newline at end of file + database: "pdf_templates" \ No newline at end of file From 46e1260d45a34fa14089f62094a692c531e52ca0 Mon Sep 17 00:00:00 2001 From: Akkahshh Date: Wed, 14 May 2025 00:19:11 +0530 Subject: [PATCH 22/29] Change config model --- service/main.go | 10 ++++++---- service/model/config.go | 29 +++++++---------------------- 2 files changed, 13 insertions(+), 26 deletions(-) diff --git a/service/main.go b/service/main.go index affe4a5..5d3bfb8 100644 --- a/service/main.go +++ b/service/main.go @@ -18,7 +18,7 @@ import ( func main() { ctx := context.Background() - config, err := config.Load("/app/espresso/configs") + config, err := config.Load() if err != nil { log.Fatalf("Error loading config: %v", err) } @@ -31,8 +31,9 @@ func main() { log.Printf("File storage type: %s", config.FileStorageConfig.StorageType) tabpool := config.BrowserConfig.TabPool + browserPath := config.AppConfig.RodBrowserBin - if err := browser_manager.Init(ctx, tabpool); err != nil { + if err := browser_manager.Init(ctx, tabpool, browserPath); err != nil { log.Fatalf("Failed to initialize browser: %v", err) } @@ -49,8 +50,9 @@ func main() { // Wrap the entire mux with the CORS middleware corsHandler := enableCORS(mux) - log.Println("Starting PDF client server on :8081") - if err := http.ListenAndServe(":8081", corsHandler); err != nil { + port := fmt.Sprintf(":%d", config.AppConfig.ServerPort) + log.Printf("Starting PDF client server on: %s\n", port) + if err := http.ListenAndServe(port, corsHandler); err != nil { log.Fatal(err) } diff --git a/service/model/config.go b/service/model/config.go index b3cde36..418bea6 100644 --- a/service/model/config.go +++ b/service/model/config.go @@ -1,19 +1,24 @@ package model +import ( + "github.com/Zomato/espresso/lib/s3" +) + type Config struct { AppConfig AppConfig `mapstructure:"app"` TemplateStorageConfig StorageConfig `mapstructure:"template_storage"` FileStorageConfig StorageConfig `mapstructure:"file_storage"` BrowserConfig BrowserConfig `mapstructure:"browser"` WorkerPoolConfig WorkerPoolConfig `mapstructure:"workerpool"` - S3Config S3Config `mapstructure:"s3"` - AWSConfig AWSConfig `mapstructure:"aws"` + S3Config s3.Config `mapstructure:"s3"` + AWSConfig s3.AwsCredConfig `mapstructure:"aws"` CertConfig map[string]CertConfig `mapstructure:"certificates"` DBConfig DBConfig `mapstructure:"db"` } type AppConfig struct { LogLevel string `mapstructure:"log_level"` + ServerPort int `mapstructure:"server_port"` EnableUI bool `mapstructure:"enable_ui"` RodBrowserBin string `mapstructure:"rod_browser_bin"` } @@ -31,26 +36,6 @@ type WorkerPoolConfig struct { WorkerTimeoutMs int `mapstructure:"worker_timeout_ms"` } -type S3Config struct { - Endpoint string `mapstructure:"endpoint"` - Debug bool `mapstructure:"debug"` - Region string `mapstructure:"region"` - ForcePathStyle bool `mapstructure:"force_path_style"` - UploaderConcurrency int `mapstructure:"uploader_concurrency"` - UploaderPartSizeMB int64 `mapstructure:"uploader_part_size_mb"` - DownloaderConcurrency int `mapstructure:"downloader_concurrency"` - DownloaderPartSizeMB int64 `mapstructure:"downloader_part_size_mb"` - RetryMaxAttempts int `mapstructure:"retry_max_attempts"` - Bucket string `mapstructure:"bucket"` - UseCustomTransport bool `mapstructure:"use_custom_transport"` -} - -type AWSConfig struct { - AccessKeyID string `mapstructure:"access_key_id"` - SecretAccessKey string `mapstructure:"secret_access_key"` - SessionToken string `mapstructure:"session_token"` -} - type CertConfig struct { CertPath string `mapstructure:"cert_path"` KeyPath string `mapstructure:"key_path"` From 8e521b212fb5610820844bdf3cd7031413a838fc Mon Sep 17 00:00:00 2001 From: Akkahshh Agarwaal Date: Thu, 22 May 2025 11:59:05 +0530 Subject: [PATCH 23/29] Fix db host and port --- service/configs/espressoconfig.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/service/configs/espressoconfig.yaml b/service/configs/espressoconfig.yaml index beb81d2..e836b62 100644 --- a/service/configs/espressoconfig.yaml +++ b/service/configs/espressoconfig.yaml @@ -48,8 +48,8 @@ certificates: key_password: "password2" db: - host: "localhost" - port: 3308 + host: "mysql" + port: 3306 username: "pdf_user" password: "pdf_password" database: "pdf_templates" \ No newline at end of file From 1368b7e40c29cffb9aa441df0a411aba0f0ead80 Mon Sep 17 00:00:00 2001 From: Akkahshh Agarwaal Date: Thu, 22 May 2025 12:00:06 +0530 Subject: [PATCH 24/29] Delete unused viperpkg --- service/internal/pkg/viperpkg/viperpkg.go | 20 -------------------- 1 file changed, 20 deletions(-) delete mode 100644 service/internal/pkg/viperpkg/viperpkg.go diff --git a/service/internal/pkg/viperpkg/viperpkg.go b/service/internal/pkg/viperpkg/viperpkg.go deleted file mode 100644 index 84807b1..0000000 --- a/service/internal/pkg/viperpkg/viperpkg.go +++ /dev/null @@ -1,20 +0,0 @@ -package viperpkg - -import ( - "log" - - "github.com/spf13/viper" -) - -func InitConfig() { - viper.SetConfigName("espressoconfig") // File name without extension - viper.SetConfigType("yaml") // File type - // Search paths relative to where the binary runs in container - viper.AddConfigPath("/app/espresso/configs") // Main config path in container - viper.AddConfigPath("../../configs") // For local development - viper.AddConfigPath("./configs") // Fallback path for local development - - if err := viper.ReadInConfig(); err != nil { - log.Fatalf("Error reading config file: %v", err) - } -} From 20c1ebde804cf1ab40fd77512c64de20cec6e36c Mon Sep 17 00:00:00 2001 From: Akkahshh Agarwaal Date: Thu, 22 May 2025 12:01:28 +0530 Subject: [PATCH 25/29] Move config model to internal/pkg/config --- service/{model/config.go => internal/pkg/config/model.go} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename service/{model/config.go => internal/pkg/config/model.go} (99%) diff --git a/service/model/config.go b/service/internal/pkg/config/model.go similarity index 99% rename from service/model/config.go rename to service/internal/pkg/config/model.go index 418bea6..02d8df0 100644 --- a/service/model/config.go +++ b/service/internal/pkg/config/model.go @@ -1,4 +1,4 @@ -package model +package config import ( "github.com/Zomato/espresso/lib/s3" From 284445ab746056c3456094093f5db393977d262c Mon Sep 17 00:00:00 2001 From: Akkahshh Agarwaal Date: Thu, 22 May 2025 12:02:05 +0530 Subject: [PATCH 26/29] Move config logic to internal/pkg/config --- service/internal/{ => pkg}/config/config.go | 11 +++++------ service/internal/{ => pkg}/config/constant.go | 0 2 files changed, 5 insertions(+), 6 deletions(-) rename service/internal/{ => pkg}/config/config.go (80%) rename service/internal/{ => pkg}/config/constant.go (100%) diff --git a/service/internal/config/config.go b/service/internal/pkg/config/config.go similarity index 80% rename from service/internal/config/config.go rename to service/internal/pkg/config/config.go index a1b2781..32c6c07 100644 --- a/service/internal/config/config.go +++ b/service/internal/pkg/config/config.go @@ -3,11 +3,10 @@ package config import ( "fmt" - "github.com/Zomato/espresso/service/model" "github.com/spf13/viper" ) -func Load() (model.Config, error) { +func Load() (Config, error) { viper.AutomaticEnv() viper.SetDefault(CONFIG_FILE_NAME, "espressoconfig") @@ -24,20 +23,20 @@ func Load() (model.Config, error) { err := viper.ReadInConfig() if err != nil { - return model.Config{}, err + return Config{}, err } - var config model.Config + var config Config err = viper.Unmarshal(&config) if err != nil { - return model.Config{}, err + return Config{}, err } // TODO: Why are these fields set in the Dockerfile and not in the config.yaml file? config.AppConfig.EnableUI = viper.GetBool(ENABLE_UI) config.AppConfig.RodBrowserBin = viper.GetString(ROD_BROWSER_BIN) if config.AppConfig.RodBrowserBin == "" { - return model.Config{}, fmt.Errorf("environment variable %s not set", ROD_BROWSER_BIN) + return Config{}, fmt.Errorf("environment variable %s not set", ROD_BROWSER_BIN) } return config, nil diff --git a/service/internal/config/constant.go b/service/internal/pkg/config/constant.go similarity index 100% rename from service/internal/config/constant.go rename to service/internal/pkg/config/constant.go From f18c11daed196a061717b40284ce61f879ff0dfa Mon Sep 17 00:00:00 2001 From: Akkahshh Agarwaal Date: Thu, 22 May 2025 12:03:21 +0530 Subject: [PATCH 27/29] Rectify config package import path --- service/controller/pdf_generation/register.go | 6 +++--- service/main.go | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/service/controller/pdf_generation/register.go b/service/controller/pdf_generation/register.go index 16192ad..ad6132e 100644 --- a/service/controller/pdf_generation/register.go +++ b/service/controller/pdf_generation/register.go @@ -6,7 +6,7 @@ import ( "net/http" "github.com/Zomato/espresso/lib/templatestore" - "github.com/Zomato/espresso/service/model" + "github.com/Zomato/espresso/service/internal/pkg/config" ) type EspressoService struct { @@ -14,7 +14,7 @@ type EspressoService struct { FileStorageAdapter *templatestore.StorageAdapter } -func NewEspressoService(config model.Config) (*EspressoService, error) { +func NewEspressoService(config config.Config) (*EspressoService, error) { templateStorageType := config.TemplateStorageConfig.StorageType if config.AppConfig.EnableUI && templateStorageType != templatestore.StorageAdapterTypeMySQL { @@ -48,7 +48,7 @@ func NewEspressoService(config model.Config) (*EspressoService, error) { return &EspressoService{TemplateStorageAdapter: &templateStorageAdapter, FileStorageAdapter: &fileStorageAdapter}, nil } -func Register(mux *http.ServeMux, config model.Config) { +func Register(mux *http.ServeMux, config config.Config) { espressoService, err := NewEspressoService(config) if err != nil { log.Fatalf("Failed to initialize PDF service: %v", err) diff --git a/service/main.go b/service/main.go index 5d3bfb8..101f90c 100644 --- a/service/main.go +++ b/service/main.go @@ -11,7 +11,7 @@ import ( logger "github.com/Zomato/espresso/lib/logger" "github.com/Zomato/espresso/lib/workerpool" "github.com/Zomato/espresso/service/controller/pdf_generation" - "github.com/Zomato/espresso/service/internal/config" + "github.com/Zomato/espresso/service/internal/pkg/config" "github.com/Zomato/espresso/service/utils" ) From 5b99095f55fa76c4a0ae1dfdbf52b7998cd4fd28 Mon Sep 17 00:00:00 2001 From: Akkahshh Date: Tue, 3 Jun 2025 21:11:33 +0530 Subject: [PATCH 28/29] use CertificateConfig struct from lib --- service/internal/pkg/config/model.go | 25 ++++++++++--------------- 1 file changed, 10 insertions(+), 15 deletions(-) diff --git a/service/internal/pkg/config/model.go b/service/internal/pkg/config/model.go index 02d8df0..40def79 100644 --- a/service/internal/pkg/config/model.go +++ b/service/internal/pkg/config/model.go @@ -1,19 +1,20 @@ package config import ( + "github.com/Zomato/espresso/lib/certmanager" "github.com/Zomato/espresso/lib/s3" ) type Config struct { - AppConfig AppConfig `mapstructure:"app"` - TemplateStorageConfig StorageConfig `mapstructure:"template_storage"` - FileStorageConfig StorageConfig `mapstructure:"file_storage"` - BrowserConfig BrowserConfig `mapstructure:"browser"` - WorkerPoolConfig WorkerPoolConfig `mapstructure:"workerpool"` - S3Config s3.Config `mapstructure:"s3"` - AWSConfig s3.AwsCredConfig `mapstructure:"aws"` - CertConfig map[string]CertConfig `mapstructure:"certificates"` - DBConfig DBConfig `mapstructure:"db"` + AppConfig AppConfig `mapstructure:"app"` + TemplateStorageConfig StorageConfig `mapstructure:"template_storage"` + FileStorageConfig StorageConfig `mapstructure:"file_storage"` + BrowserConfig BrowserConfig `mapstructure:"browser"` + WorkerPoolConfig WorkerPoolConfig `mapstructure:"workerpool"` + S3Config s3.Config `mapstructure:"s3"` + AWSConfig s3.AwsCredConfig `mapstructure:"aws"` + CertConfig map[string]certmanager.CertificateConfig `mapstructure:"certificates"` + DBConfig DBConfig `mapstructure:"db"` } type AppConfig struct { @@ -36,12 +37,6 @@ type WorkerPoolConfig struct { WorkerTimeoutMs int `mapstructure:"worker_timeout_ms"` } -type CertConfig struct { - CertPath string `mapstructure:"cert_path"` - KeyPath string `mapstructure:"key_path"` - KeyPassword string `mapstructure:"key_password"` -} - type DBConfig struct { Host string `mapstructure:"host"` Port int `mapstructure:"port"` From 35cbf9a7f12d23706ea3005becf8c1f2fbfa9519 Mon Sep 17 00:00:00 2001 From: Akkahshh Date: Tue, 3 Jun 2025 21:11:59 +0530 Subject: [PATCH 29/29] Add credential store to load signing credentials at startup --- service/configs/espressoconfig.yaml | 8 +- .../pdf_generation/pdf_generation.go | 19 ++-- service/controller/pdf_generation/register.go | 13 ++- service/internal/config/config.go | 44 ---------- service/internal/config/constant.go | 10 --- .../service/generateDoc/generatePdf.go | 88 ++++++------------- service/main.go | 1 + 7 files changed, 46 insertions(+), 137 deletions(-) delete mode 100644 service/internal/config/config.go delete mode 100644 service/internal/config/constant.go diff --git a/service/configs/espressoconfig.yaml b/service/configs/espressoconfig.yaml index e836b62..cd64192 100644 --- a/service/configs/espressoconfig.yaml +++ b/service/configs/espressoconfig.yaml @@ -39,13 +39,9 @@ aws: certificates: cert1: - cert_path: "./inputfiles/certificates/cert.pem" - key_path: "./inputfiles/certificates/key_pkcs8_encrypted.pem" + cert_file_path: "./inputfiles/certificates/cert.pem" + key_file_path: "./inputfiles/certificates/key_pkcs8_encrypted.pem" key_password: "test" - cert2: - cert_path: "./certificates/certificate2.pem" - key_path: "./certificates/pirvatekey2.key" - key_password: "password2" db: host: "mysql" diff --git a/service/controller/pdf_generation/pdf_generation.go b/service/controller/pdf_generation/pdf_generation.go index 97c4a8a..e33c577 100644 --- a/service/controller/pdf_generation/pdf_generation.go +++ b/service/controller/pdf_generation/pdf_generation.go @@ -14,7 +14,6 @@ import ( "github.com/Zomato/espresso/service/internal/pkg/httppkg" "github.com/Zomato/espresso/service/internal/service/generateDoc" svcUtils "github.com/Zomato/espresso/service/utils" - "github.com/spf13/viper" ) func (s *EspressoService) GeneratePDF(w http.ResponseWriter, r *http.Request) { @@ -56,7 +55,7 @@ func (s *EspressoService) GeneratePDF(w http.ResponseWriter, r *http.Request) { generatePdfReq.SignParams = req.SignParams } - err = generateDoc.GeneratePDF(ctx, generatePdfReq, s.TemplateStorageAdapter, s.FileStorageAdapter) + err = generateDoc.GeneratePDF(ctx, generatePdfReq, s.TemplateStorageAdapter, s.FileStorageAdapter, s.CredentialStore) if err != nil { svcUtils.Logger.Error(ctx, "error in generating pdf :: %v", err, nil) httppkg.RespondWithError(w, "Failed to generate PDF: "+err.Error(), http.StatusInternalServerError) @@ -143,7 +142,7 @@ func (s *EspressoService) GeneratePDFStream(w http.ResponseWriter, r *http.Reque if pdfReq.SignPdf { generatePdfReq.SignParams = &generateDoc.SignParams{ SignPdf: true, - CertConfigKey: "digital_certificates.cert1", // certificate details are stored in config file + CertConfigKey: "cert1", // certificate details are stored in config file } } @@ -155,16 +154,8 @@ func (s *EspressoService) GeneratePDFStream(w http.ResponseWriter, r *http.Reque httppkg.RespondWithError(w, "Failed to get file storage adapter: "+err.Error(), http.StatusExpectationFailed) return } - templateStorageAdapter, err := templatestore.TemplateStorageAdapterFactory(&templatestore.StorageConfig{ - StorageType: "mysql", - MysqlDSN: viper.GetString("mysql.dsn"), - }) - if err != nil { - svcUtils.Logger.Error(ctx, "error in getting file storage adapter :: %v", err, nil) - httppkg.RespondWithError(w, "Failed to get file storage adapter: "+err.Error(), http.StatusExpectationFailed) - return - } - err = generateDoc.GeneratePDF(ctx, generatePdfReq, &templateStorageAdapter, &fileStorageAdapter) + + err = generateDoc.GeneratePDF(ctx, generatePdfReq, s.TemplateStorageAdapter, &fileStorageAdapter, s.CredentialStore) if err != nil { svcUtils.Logger.Error(ctx, "error in generating pdf stream:: %v", err, nil) httppkg.RespondWithError(w, "Failed to generate PDF stream: "+err.Error(), http.StatusInternalServerError) @@ -243,7 +234,7 @@ func (s *EspressoService) SignPDF(w http.ResponseWriter, r *http.Request) { httppkg.RespondWithError(w, err.Error(), http.StatusBadRequest) return } - err := generateDoc.SignPDF(ctx, signPDFDto, s.FileStorageAdapter) + err := generateDoc.SignPDF(ctx, signPDFDto, s.FileStorageAdapter, s.CredentialStore) if err != nil { svcUtils.Logger.Error(ctx, "error in signing pdf :: : %v", err, nil) httppkg.RespondWithError(w, "Failed to sign PDF: "+err.Error(), http.StatusInternalServerError) diff --git a/service/controller/pdf_generation/register.go b/service/controller/pdf_generation/register.go index ad6132e..58008b7 100644 --- a/service/controller/pdf_generation/register.go +++ b/service/controller/pdf_generation/register.go @@ -5,6 +5,7 @@ import ( "log" "net/http" + "github.com/Zomato/espresso/lib/certmanager" "github.com/Zomato/espresso/lib/templatestore" "github.com/Zomato/espresso/service/internal/pkg/config" ) @@ -12,6 +13,7 @@ import ( type EspressoService struct { TemplateStorageAdapter *templatestore.StorageAdapter FileStorageAdapter *templatestore.StorageAdapter + CredentialStore *certmanager.CredentialStore } func NewEspressoService(config config.Config) (*EspressoService, error) { @@ -46,7 +48,16 @@ func NewEspressoService(config config.Config) (*EspressoService, error) { return nil, err } - return &EspressoService{TemplateStorageAdapter: &templateStorageAdapter, FileStorageAdapter: &fileStorageAdapter}, nil + credentialStore, err := certmanager.NewCredentialStore(config.CertConfig) + if err != nil { + return nil, err + } + + return &EspressoService{ + TemplateStorageAdapter: &templateStorageAdapter, + FileStorageAdapter: &fileStorageAdapter, + CredentialStore: credentialStore, + }, nil } func Register(mux *http.ServeMux, config config.Config) { espressoService, err := NewEspressoService(config) diff --git a/service/internal/config/config.go b/service/internal/config/config.go deleted file mode 100644 index a1b2781..0000000 --- a/service/internal/config/config.go +++ /dev/null @@ -1,44 +0,0 @@ -package config - -import ( - "fmt" - - "github.com/Zomato/espresso/service/model" - "github.com/spf13/viper" -) - -func Load() (model.Config, error) { - viper.AutomaticEnv() - - viper.SetDefault(CONFIG_FILE_NAME, "espressoconfig") - viper.SetDefault(CONFIG_FILE_TYPE, "yaml") - viper.SetDefault(CONFIG_FILE_PATH, "/app/espresso/configs") - - viper.SetConfigName(viper.GetString(CONFIG_FILE_NAME)) // File name without extension - viper.SetConfigType(viper.GetString(CONFIG_FILE_TYPE)) // File type - - // Search paths relative to where the binary runs in container - viper.AddConfigPath(CONFIG_FILE_PATH) // Main config path in container - viper.AddConfigPath("../../configs") // For local development - viper.AddConfigPath("./configs") // Fallback path for local development - - err := viper.ReadInConfig() - if err != nil { - return model.Config{}, err - } - - var config model.Config - err = viper.Unmarshal(&config) - if err != nil { - return model.Config{}, err - } - - // TODO: Why are these fields set in the Dockerfile and not in the config.yaml file? - config.AppConfig.EnableUI = viper.GetBool(ENABLE_UI) - config.AppConfig.RodBrowserBin = viper.GetString(ROD_BROWSER_BIN) - if config.AppConfig.RodBrowserBin == "" { - return model.Config{}, fmt.Errorf("environment variable %s not set", ROD_BROWSER_BIN) - } - - return config, nil -} diff --git a/service/internal/config/constant.go b/service/internal/config/constant.go deleted file mode 100644 index 92dc8ca..0000000 --- a/service/internal/config/constant.go +++ /dev/null @@ -1,10 +0,0 @@ -package config - -const ( - CONFIG_FILE_NAME = "CONFIG_FILE_NAME" - CONFIG_FILE_TYPE = "CONFIG_FILE_TYPE" - CONFIG_FILE_PATH = "CONFIG_FILE_PATH" - - ENABLE_UI = "ENABLE_UI" - ROD_BROWSER_BIN = "ROD_BROWSER_BIN" -) diff --git a/service/internal/service/generateDoc/generatePdf.go b/service/internal/service/generateDoc/generatePdf.go index 1d2acd3..52c73c8 100644 --- a/service/internal/service/generateDoc/generatePdf.go +++ b/service/internal/service/generateDoc/generatePdf.go @@ -5,7 +5,6 @@ import ( "context" "fmt" "io" - "sync" "time" "github.com/Zomato/espresso/lib/browser_manager" @@ -13,8 +12,6 @@ import ( "github.com/Zomato/espresso/lib/renderer" "github.com/Zomato/espresso/lib/signer" "github.com/Zomato/espresso/lib/templatestore" - "github.com/Zomato/espresso/lib/workerpool" - "github.com/spf13/viper" svcUtils "github.com/Zomato/espresso/service/utils" "github.com/go-rod/rod/lib/proto" @@ -24,7 +21,13 @@ import ( // If signing is enabled, it will load the signing credentials in parallel and sign the PDF before storing it. // The generated PDF is stored in the file store with the provided output file path. // The function returns an error if anything goes wrong during generation, signing, or storage of the PDF. -func GeneratePDF(ctx context.Context, req *PDFDto, templateStoreAdapter *templatestore.StorageAdapter, fileStoreAdapter *templatestore.StorageAdapter) error { +func GeneratePDF( + ctx context.Context, + req *PDFDto, + templateStoreAdapter *templatestore.StorageAdapter, + fileStoreAdapter *templatestore.StorageAdapter, + credentialStore *certmanager.CredentialStore, +) error { startTime := time.Now() @@ -32,36 +35,18 @@ func GeneratePDF(ctx context.Context, req *PDFDto, templateStoreAdapter *templat content := req.Content viewPortConfig := req.ViewPort pdfParams := req.PdfParams - viewPort := getViewPort(viewPortConfig) - // Start loading credentials in parallel if signing is enabled - var credWg sync.WaitGroup - var credErr error + var credentials *certmanager.SigningCredentials var pdfReader io.Reader - toBeSigned := false - if req.SignParams != nil && req.SignParams.SignPdf { - toBeSigned = true - } + toBeSigned := req.SignParams != nil && req.SignParams.SignPdf if toBeSigned { - certConfig := &certmanager.CertificateConfig{ - CertFilePath: viper.GetString(req.SignParams.CertConfigKey + ".cert_filepath"), - KeyFilePath: viper.GetString(req.SignParams.CertConfigKey + ".key_filepath"), - KeyPassword: viper.GetString(req.SignParams.CertConfigKey + ".key_password"), - } - credWg.Add(1) - err := workerpool.Pool().SubmitTask( - func(args ...interface{}) { - defer credWg.Done() - ctxArg := args[0].(context.Context) - credentials, credErr = certmanager.LoadSigningCredentials(ctxArg, certConfig) - }, - ctx, - ) - - if err != nil { - return fmt.Errorf("failed to submit credential loading task: %v", err) + credKey := req.SignParams.CertConfigKey + var exists bool + credentials, exists = credentialStore.GetCredential(credKey) + if !exists { + return fmt.Errorf("signing credentials not found for key: %s", credKey) } } @@ -97,12 +82,6 @@ func GeneratePDF(ctx context.Context, req *PDFDto, templateStoreAdapter *templat duration = time.Since(startTime) if toBeSigned { - credWg.Wait() - - if credErr != nil { - return fmt.Errorf("failed to load signing credentials: %v", credErr) - } - signedPDF, err := signer.SignPdfStream(ctx, pdf, credentials.Certificate, credentials.PrivateKey) if err != nil { return fmt.Errorf("failed to sign pdf using SignPdfStream: %v", err) @@ -191,7 +170,12 @@ func getViewPort(viewPort *ViewportConfig) *browser_manager.ViewportConfig { return viewSettings } -func SignPDF(ctx context.Context, req *SignPDFDto, fileStoreAdapter *templatestore.StorageAdapter) error { +func SignPDF( + ctx context.Context, + req *SignPDFDto, + fileStoreAdapter *templatestore.StorageAdapter, + credentialStore *certmanager.CredentialStore, +) error { reqId := req.ReqId svcUtils.Logger.Info(ctx, "SignPDF called ", map[string]any{"req id": reqId}) @@ -205,38 +189,18 @@ func SignPDF(ctx context.Context, req *SignPDFDto, fileStoreAdapter *templatesto if err != nil { return fmt.Errorf("failed to get input file: %v", err) } - // Start loading credentials in parallel if signing is enabled - var credWg sync.WaitGroup - var credErr error + var credentials *certmanager.SigningCredentials var pdfReader io.Reader if req.SignParams.SignPdf { - credWg.Add(1) - certConfig := &certmanager.CertificateConfig{ - CertFilePath: viper.GetString(req.SignParams.CertConfigKey + ".cert_filepath"), - KeyFilePath: viper.GetString(req.SignParams.CertConfigKey + ".key_filepath"), - KeyPassword: viper.GetString(req.SignParams.CertConfigKey + ".key_password"), - } - err := workerpool.Pool().SubmitTask( - func(args ...interface{}) { - defer credWg.Done() - ctxArg := args[0].(context.Context) - credentials, credErr = certmanager.LoadSigningCredentials(ctxArg, certConfig) - }, - ctx, - ) - - if err != nil { - return fmt.Errorf("failed to submit credential loading task: %v", err) + credKey := req.SignParams.CertConfigKey + var exists bool + credentials, exists = credentialStore.GetCredential(credKey) + if !exists { + return fmt.Errorf("signing credentials not found for key: %s", credKey) } - } - if req.SignParams.SignPdf { - credWg.Wait() - if credErr != nil { - return fmt.Errorf("failed to load signing credentials: %v", credErr) - } // convert pdfreader to *rod.StreamReader signedPDF, err := signer.SignPdfStream(ctx, freader, credentials.Certificate, credentials.PrivateKey) if err != nil { diff --git a/service/main.go b/service/main.go index 101f90c..e785d8d 100644 --- a/service/main.go +++ b/service/main.go @@ -2,6 +2,7 @@ package main import ( "context" + "fmt" "log" "net/http" "time"