Correctly replace sampledata file path for database connections too (#254)

* Fix for format

* Handle unpacking asar

* External asar

* Add mkdir

* One more missing asar external

* Copy in asar

* Add sampledata

Author: eatonphil

desktop/scripts/desktop.build

@@ -6,8 +6,8 @@ setenv_default VERSION "development"
 cd runner && go build -trimpath -buildmode=pie -mod=readonly -modcacherw -ldflags="-s -w -X main.VERSION={VERSION}" -o ../build/go_desktop_runner{required_ext} cmd/main.go
 
 yarn esbuild desktop/preload.ts --external:electron --sourcemap --bundle --outfile=build/preload.js
-yarn esbuild desktop/runner.ts --bundle --platform=node --sourcemap --external:better-sqlite3 --external:react-native-fs --external:react-native-fetch-blob "--external:@elastic/elasticsearch" "--external:wasm-brotli" --external:prometheus-query --external:snowflake-sdk --external:ssh2 --external:ssh2-promise --external:ssh2-sftp-client --external:cpu-features --external:electron --target=node10.4 --outfile=build/desktop_runner.js
-yarn esbuild desktop/app.ts --bundle --platform=node --sourcemap --external:better-sqlite3 --external:react-native-fs --external:react-native-fetch-blob "--external:@elastic/elasticsearch" "--external:wasm-brotli" --external:prometheus-query --external:snowflake-sdk --external:ssh2 --external:ssh2-promise --external:ssh2-sftp-client --external:cpu-features --external:electron --target=node10.4 --outfile=build/desktop.js
+yarn esbuild desktop/runner.ts --bundle --platform=node --sourcemap --external:better-sqlite3 --external:react-native-fs --external:asar --external:react-native-fetch-blob "--external:@elastic/elasticsearch" "--external:wasm-brotli" --external:prometheus-query --external:snowflake-sdk --external:ssh2 --external:ssh2-promise --external:ssh2-sftp-client --external:cpu-features --external:electron --target=node10.4 --outfile=build/desktop_runner.js
+yarn esbuild desktop/app.ts --bundle --platform=node --sourcemap --external:better-sqlite3 --external:react-native-fs --external:react-native-fetch-blob "--external:@elastic/elasticsearch" "--external:wasm-brotli" --external:prometheus-query --external:snowflake-sdk --external:ssh2 --external:ssh2-promise --external:asar --external:ssh2-sftp-client --external:cpu-features --external:electron --target=node10.4 --outfile=build/desktop.js
 
 rm -rf build/migrations
 cp -r desktop/migrations build/migrations

desktop/store.ts

@@ -1,3 +1,4 @@
+import asar from 'asar';
 import * as sqlite3 from 'better-sqlite3';
 import { Buffer } from 'buffer';
 import fs from 'fs';
@@ -7,6 +8,7 @@ import { getPath } from '../shared/object';
 import { GetProjectRequest, MakeProjectRequest } from '../shared/rpc';
 import {
   ConnectorInfo,
+  DatabaseConnectorInfo,
   DatabasePanelInfo,
   doOnEncryptFields,
   Encrypt,
@@ -185,7 +187,11 @@ export class Store {
           const name = f.slice(0, f.length - ('.' + PROJECT_EXTENSION).length);
           return { createdAt, name };
         });
-      files.sort();
+      // Sort timestamp DESC
+      files.sort(
+        (a, b) =>
+          new Date(b.createdAt).valueOf() - new Date(a.createdAt).valueOf()
+      );
       return files;
     },
   };
@@ -378,17 +384,56 @@ GROUP BY panel_id
     },
   };
 
+  // Example: /private/var/folders/l0/51ds3d1d2214wb1y0vbtl5qr0000gn/T/AppTranslocation/6AC58880-BE22-4AB7-8006-47D9764BC590/d/DataStation Desktop CE.app/Contents/Resources/app.asar/sampledata/nginx_logs.jsonl
+  unmangleAsar(file: string): string {
+    const asarName = 'app.asar';
+    if (!file.includes(asarName)) {
+      return file;
+    }
+
+    // Since Go reads from the filesystem it doesn't look into the asar that is used in release builds. So we need to extract it if it doesn't exist.
+    const [asarParent, fileName] = file.split(asarName);
+    const asarFile = asarParent + asarName;
+    const newFile = path.join(asarParent, fileName);
+    // TODO: if these files change then checksum will be needed
+    if (!fs.existsSync(newFile)) {
+      fs.mkdirSync(path.dirname(newFile), { recursive: true });
+      fs.writeFileSync(
+        newFile,
+        asar.extractFile(asarFile, fileName.slice(1) /* drop leading / */)
+      );
+    }
+
+    return newFile;
+  }
+
   cleanupSampleProject(sampleProject: ProjectState) {
     for (const page of sampleProject.pages || []) {
       for (const panel of page.panels || []) {
         if (panel.type === 'file') {
           const fp = panel as FilePanelInfo;
           if (fp.file.name.startsWith('sampledata')) {
-            fp.file.name = path.join(CODE_ROOT, fp.file.name);
+            fp.file.name = this.unmangleAsar(
+              path.join(CODE_ROOT, fp.file.name)
+            );
           }
         }
       }
     }
+
+    for (const con of sampleProject.connectors || []) {
+      if (con.type === 'database') {
+        const dc = con as DatabaseConnectorInfo;
+        if (
+          dc.database.type === 'sqlite' &&
+          dc.database.database.startsWith('sampledata')
+        ) {
+          dc.database.database = this.unmangleAsar(
+            path.join(CODE_ROOT, dc.database.database)
+          );
+        }
+      }
+    }
   }
 
   makeProjectHandler: MakeProjectHandler = {

package.json

@@ -28,6 +28,7 @@
   "dependencies": {
     "@tabler/icons": "^1.68.0",
     "ace-builds": "^1.5.0",
+    "asar": "^3.1.0",
     "better-sqlite3": "^7.5.3",
     "chart.js": "^3.7.1",
     "cookie-parser": "^1.4.6",

runner/scripts/runner_test.build

@@ -1,3 +1,3 @@
-yarn esbuild desktop/runner.ts --bundle --platform=node --sourcemap --external:sqlite3 --external:react-native-fs --external:react-native-fetch-blob --external:oracledb "--external:@elastic/elasticsearch" "--external:wasm-brotli" --external:prometheus-query --external:snowflake-sdk --external:ssh2 --external:ssh2-promise --external:ssh2-sftp-client --external:cpu-features --external:electron --external:pg-native --target=node10.4 --outfile=build/desktop_runner.js
+yarn esbuild desktop/runner.ts --bundle --platform=node --sourcemap --external:sqlite3 --external:asar --external:react-native-fs --external:react-native-fetch-blob --external:oracledb "--external:@elastic/elasticsearch" "--external:wasm-brotli" --external:prometheus-query --external:snowflake-sdk --external:ssh2 --external:ssh2-promise --external:ssh2-sftp-client --external:cpu-features --external:electron --external:pg-native --target=node10.4 --outfile=build/desktop_runner.js
 
 cd runner/cmd && go test -trimpath -mod=readonly -modcacherw -ldflags="-s -w" -race -o ../../build/go_desktop_runner_test{required_ext} -coverpkg="." -c -tags testrunmain

scripts/build.py

@@ -168,11 +168,15 @@ def eval_line(line):
         quote_line(line, show=True)
         if not what:
             return
-        with open(to, 'rb+') as to:
-            current = to.read()
-            to.seek(0)
-            to.write(what.encode()+b'\n'+current)
-            to.truncate()
+        try:
+            with open(to, 'rb+') as to:
+                current = to.read()
+                to.seek(0)
+                to.write(what.encode()+b'\n'+current)
+                to.truncate()
+        except FileNotFoundError:
+            with open(to, 'w') as to:
+                to.write(what)
         return
 
     quote_line(line, show=True)

server/scripts/release.build

@@ -13,9 +13,14 @@ mkdir datastation
 mkdir datastation/node_modules
 mv build datastation/
 
+# Bring in sampledata
+cp -r sampledata datastation/sampledata
+
 # Bring in node_modules
-cp -r node_modules/electron datastation/node_modules/electron
-cp -r node_modules/better-sqlite3 datastation/node_modules/better-sqlite3
+prepend "{{}}" datastation/package.json
+cd datastation && yarn add asar electron better-sqlite3
+rm datastation/yarn.lock
+rm datastation/package.json
 # Only need stubs not full chrome install
 rm -rf datastation/node_modules/electron/dist
 

server/scripts/server.build

@@ -6,8 +6,8 @@ yarn build-ui
 
 cd runner && go build -trimpath -buildmode=pie -mod=readonly -modcacherw -ldflags="-s -w" -o ../build/go_server_runner{required_ext} ./cmd/main.go
 
-yarn esbuild server/runner.ts --sourcemap --platform=node --bundle --target=node10.4 --external:sqlite3 --external:react-native-fs --external:react-native-fetch-blob --external:cpu-features --external:electron --outfile=build/server_runner.js
-yarn esbuild server/index.ts --sourcemap --platform=node --bundle --external:sqlite3 --external:react-native-fs --external:react-native-fetch-blob --external:cpu-features --external:electron --outfile=build/server.js
+yarn esbuild server/runner.ts --sourcemap --platform=node --bundle --target=node10.4 --external:sqlite3 --external:asar --external:react-native-fs --external:react-native-fetch-blob --external:cpu-features --external:electron --outfile=build/server_runner.js
+yarn esbuild server/index.ts --sourcemap --platform=node --bundle --external:sqlite3 --external:react-native-fs --external:asar --external:react-native-fetch-blob --external:cpu-features --external:electron --outfile=build/server.js
 
 prepend "global.DS_UI_TITLE='DataStation Server CE';" build/server.js
 

ui/style.css

@@ -17,6 +17,7 @@
   /* Generic settings */
   --sans-serif: 'San Francisco', 'Gadugi', sans-serif;
   --margin-base: 8px;
+  --label-width: 100px;
 }
 
 * {
@@ -201,7 +202,7 @@ label,
   .select-label {
     display: inline-block;
     text-align: right;
-    width: 100px;
+    width: var(--label-width);
     padding-right: calc(2 * var(--margin-base));
     overflow-wrap: break-word;
   }
@@ -599,6 +600,11 @@ input + input[type='file'] {
   .connector-body,
   .server-body {
     overflow-x: auto;
+
+    input,
+    select {
+      max-width: 250px;
+    }
   }
 
   .radio-label,
@@ -1121,6 +1127,10 @@ canvas {
   width: 500px;
   margin: 0 auto;
   margin-top: calc(2 * var(--margin-base));
+
+  .select select {
+    max-width: 300px !important;
+  }
 }
 
 .card--full {