introduce Testcontainers & Testcontainers/mysql to use MySQL instead of in-memory SQLite.

Signed-off-by: ntny <[email protected]>
Signed-off-by: arpechenin <[email protected]>

Author: arpechenin

backend/src/apiserver/model/experiment.go

@@ -14,6 +14,10 @@
 
 package model
 
+func (Experiment) TableName() string {
+	return "experiments"
+}
+
 type Experiment struct {
 	UUID                  string       `gorm:"column:UUID; not null; primary_key;"`
 	Name                  string       `gorm:"column:Name; not null; unique_index:idx_name_namespace;"`

backend/src/apiserver/model/resource_reference.go

@@ -146,6 +146,10 @@ type ResourceType string
 // The relationship between two resource objects.
 type Relationship string
 
+func (ResourceReference) TableName() string {
+	return "resource_references"
+}
+
 // Resource reference table models the relationship between resources in a loosely coupled way.
 type ResourceReference struct {
 	// ID of the resource object

backend/src/apiserver/model/run.go

@@ -318,6 +318,10 @@ type RunDetails struct {
 	TaskDetails             []*Task
 }
 
+func (RunMetric) TableName() string {
+	return "run_metrics"
+}
+
 type RunMetric struct {
 	RunUUID     string  `gorm:"column:RunUUID; not null; primary_key;"`
 	NodeID      string  `gorm:"column:NodeID; not null; primary_key;"`

backend/src/apiserver/storage/db_fake.go

@@ -15,20 +15,34 @@
 package storage
 
 import (
+	"database/sql"
+	"fmt"
 	"github.com/golang/glog"
 	"github.com/jinzhu/gorm"
 	"github.com/kubeflow/pipelines/backend/src/apiserver/model"
 	"github.com/kubeflow/pipelines/backend/src/common/util"
 	_ "github.com/mattn/go-sqlite3"
 )
 
-func NewFakeDB() (*DB, error) {
+func NewInMemoryTestDB() (*DB, error) {
 	// Initialize GORM
 	db, err := gorm.Open("sqlite3", ":memory:")
 	if err != nil {
 		return nil, util.Wrap(err, "Could not create the GORM database")
 	}
 	// Create tables
+	dbWithMigration, err := migrate(db)
+	if err != nil {
+		return nil, util.Wrap(err, "Could not migrate the database")
+	}
+	return NewDB(dbWithMigration, NewSQLiteDialect()), nil
+}
+
+func migrate(db *gorm.DB) (*sql.DB, error) {
+	if db == nil {
+		return nil, fmt.Errorf("db cannot be nil")
+	}
+	// Create tables
 	db.AutoMigrate(
 		&model.Experiment{},
 		&model.Job{},
@@ -41,13 +55,13 @@ func NewFakeDB() (*DB, error) {
 		&model.DBStatus{},
 		&model.DefaultExperiment{},
 	)
-	return NewDB(db.DB(), NewSQLiteDialect()), nil
+	return db.DB(), nil
 }
 
 func NewFakeDBOrFatal() *DB {
-	db, err := NewFakeDB()
+	db, err := NewInMemoryTestDB()
 	if err != nil {
-		glog.Fatalf("The fake DB doesn't create successfully. Fail fast")
+		glog.Fatalf("The fake DB doesn't create successfully. Fail fast: %v", err)
 	}
 	return db
 }

backend/src/apiserver/storage/mysql_fake.go

@@ -0,0 +1,48 @@
+package storage
+
+import (
+	"context"
+	"fmt"
+	"github.com/golang/glog"
+	"github.com/jinzhu/gorm"
+	_ "github.com/jinzhu/gorm/dialects/mysql"
+	"github.com/kubeflow/pipelines/backend/src/common/util"
+	mysqlcontainer "github.com/testcontainers/testcontainers-go/modules/mysql"
+)
+
+func NewMySqlDBOrFatal(container *mysqlcontainer.MySQLContainer, ctx context.Context) *DB {
+	gormDb, err := OpenMysqlDb(ctx, container)
+	if err != nil {
+		glog.Fatalf(err.Error())
+	}
+	db, err := ApplyMigration(gormDb)
+	return db
+}
+
+func LaunchMysqlContainer(ctx context.Context) (*mysqlcontainer.MySQLContainer, error) {
+	container, err := mysqlcontainer.Run(ctx, "mysql:8.0")
+	if err != nil {
+		return nil, fmt.Errorf("error during creating mysql container for testing: %v", err)
+	}
+	return container, nil
+}
+
+func OpenMysqlDb(ctx context.Context, container *mysqlcontainer.MySQLContainer) (*gorm.DB, error) {
+	connString, err := container.ConnectionString(ctx)
+	if err != nil {
+		return nil, fmt.Errorf("error during creating mysql connection string for testing: %v", err)
+	}
+	db, err := gorm.Open("mysql", connString)
+	if err != nil {
+		return nil, util.Wrap(err, "Could not create the GORM mysql database for testing")
+	}
+	return db, nil
+}
+
+func ApplyMigration(db *gorm.DB) (*DB, error) {
+	dbWithMigration, err := migrate(db)
+	if err != nil {
+		return nil, util.Wrap(err, "Could not migrate the database")
+	}
+	return NewDB(dbWithMigration, NewMySQLDialect()), nil
+}