Merge branch 'Develop'

Author: GodOfAllBeings

API/Endpoints.js

@@ -1,39 +1,33 @@
 import fs from 'fs';
 import path from "path";
 import crypto from "crypto";
-
-import {
-  getMinerPath,
-  getMinerFile,
-} from "../App/ConfigUnpacker.js";
+import { getForeignMiner } from "./RequestHandlers.js";
+import { initSingleVenv, getVenvStatusDeleteIfDone } from "../App/PyVenvHelper.js";
+import { getMinerPath, getMinerFile } from "../App/ConfigUnpacker.js";
 import {
   stopProcess,
   getStatusDeleteIfDone,
   getProcessStatusList,
   processStart,
 } from "../App/Wrapper.js";
-import {
-  getForeignMiner,
-} from "./RequestHandlers.js";
-import {
-  initSingleVenv,
-  getVenvStatusDeleteIfDone,
-} from "../App/PyVenvHelper.js";
 
 const port = 5000; // The host port express will listen on.
 
 export function initEndpoints(app, configList) {
-  app.get("/", function (req, res) {
+
+  app.get("/", function (req, res) { 
     res.send("Default page");
   });
-  app.get(`/ping`, function (req, res) {
+
+  app.get(`/ping`, function (req, res) { // return pong
     res.send("pong");
   });
-  app.get(`/configurations`, function (req, res) {
+
+  app.get(`/configurations`, function (req, res) { // return all configurations
     res.send(configList);
   });
-  // Endpoint to return algorithm file that needs to be shadowed.
-  app.get(`/shadow/:minerId`, function (req, res) {
+  
+  app.get(`/shadow/:minerId`, function (req, res) { // return algorthm file of a miner 
     const minerId = req.params.minerId;
     console.log(`Getting a request on /shadow/${minerId}`);
     let requestedConfig = configList.find(miner => miner.MinerId == minerId);
@@ -52,37 +46,29 @@ export function initEndpoints(app, configList) {
       return;
     }
 
-    // else, all is good, return file
-    res.setHeader('Content-disposition', 'attachment; filename=shadow-miner');
-
-    // TODO: Consider if these are necessary? Leave them out for now, since we're testing with a .py script.
-    // res.setHeader('Content-type', 'application/x-msdownload');      //for exe file
-    // res.setHeader('Content-type', 'application/x-rar-compressed');  //for rar file
+    // else, no issues, return file
+    res.setHeader('Content-disposition', 'attachment; filename=shadow-miner'); 
     var file = fs.createReadStream(pathToFile);
-    file.pipe(res); //send file
+    file.pipe(res);
   });
-  // Endpoint to return requirements file that needs to be shadowed.
-  app.get(`/shadow/requirements/:minerId`, function (req, res) {
+
+  app.get(`/shadow/requirements/:minerId`, function (req, res) { // Return requirements file of a miner
     console.log(`Getting a request on /shadow/requirements/${req.params.minerId}`);
     let requestedConfig = configList.find(miner => miner.MinerId == req.params.minerId);
     if(!requestedConfig.Shadow) res.status(400).send(`Invalid request, cannot shadow Miner with id \"${requestedConfig.MinerId}\" and label: \"${requestedConfig.MinerLabel}\".`);
     else {
       res.setHeader('Content-disposition', 'attachment; filename=shadow-miner');
-
-      // TODO: Consider if these are necessary? Leave them out for now, since we're testing with a .py script.
-      // res.setHeader('Content-type', 'application/x-msdownload');      //for exe file
-      // res.setHeader('Content-type', 'application/x-rar-compressed');  //for rar file
-      const pathToFile = path.join(getMinerPath(requestedConfig), "requirements.txt"); // TODO: Name of the file shouldn't just be hardcoded in here.
+      const pathToFile = path.join(getMinerPath(requestedConfig), "requirements.txt");
       if(!fs.existsSync(pathToFile)) res.status(404).send(`Unable to find requirements file for requested miner.`);
       else {
         var file = fs.createReadStream(pathToFile);
-        file.pipe(res); //send file
+        file.pipe(res);
       }
     }
   });
-  // Initiate shadow process - request foreign miner on "shadow/:minerId" to get the foreign miner .exe/script
-  app.post(`/shadow`, async function (req, res) {
-    console.log(`Getting a request on /shadow`);
+
+  app.post(`/shadow`, async function (req, res) { // Initiate cloning process: request foreign miner on "shadow/:minerId" to get the foreign miner .exe/script
+    console.log(`\nGetting a request on /shadow`);
 
     const venvInitId = crypto.randomUUID();
     let body = await req.body;
@@ -98,55 +84,54 @@ export function initEndpoints(app, configList) {
     });
   });
 
-  app.get(`/shadow/status/:venvInitId`, async function (req, res) {
+  
+  app.get(`/shadow/status/:venvInitId`, async function (req, res) { // Return status of cloning action
     let venvInitId = req.params.venvInitId;
     console.log(`Getting a request on /shadow/status for id ${venvInitId}`);
     let venvStatus = await getVenvStatusDeleteIfDone(venvInitId);
     if(venvStatus) res.status(200).send(venvStatus);
     else res.status(400).send(`No process exists with ID: ${venvInitId}`);
   });
 
+  app.post(`/miner`, async function (req, res) { // Start miner - return process id.
+    console.log("Received POST request on /miner");
+    function sendProcessId(processId, error) {
+      if(error) {
+        console.error("Error: " + error);
+        res.status(400).send(error);
+      }
+      else {
+        res.send(processId.toString());
+      }
+    } 
+    const body = await req.body;
+    const ownUrl = req.protocol + '://' + req.get('host') + req.originalUrl;
+    processStart(sendProcessId, body, ownUrl, configList);
+  });
 
-
-  app.get(`/status`, async function (req, res) {
+  app.get(`/status`, async function (req, res) { // Return status of all active miners
     console.log(`Getting a request on /status for status list`);
     res.status(200).send(getProcessStatusList());
   });
 
-  app.get(`/status/:processId`, async function (req, res) {
+  app.get(`/status/:processId`, async function (req, res) { // Return status of specific miner
     let processId = req.params.processId;
     // console.log(`Getting a request on /status for id ${processId}`);
     let statusDict = await getStatusDeleteIfDone(processId);
     if(statusDict) res.status(200).send(statusDict);
     else res.status(400).send(`No process exists with ID: ${processId}`);
   });
 
-  app.delete(`/stop/:processId`, async function (req, res) {
+  app.delete(`/stop/:processId`, async function (req, res) { // Stop miner - return confirmation.
     let processId = req.params.processId
     console.log(`Getting a request on /stop for id ${processId}`);
     let result = await stopProcess(processId);
     if(result) res.status(200).send(`Killed process with ID: ${processId}`);
     else res.status(400).send(`No active process with ID: ${processId}`);
   });
 
-  app.post(`/miner`, async function (req, res) {
-    console.log("Received POST request on /miner");
-    function sendProcessId(processId, error) {
-      if(error) {
-        console.error("Error: " + error);
-        res.status(400).send(error);
-      }
-      else {
-        // console.log(`Sending processId ${processId}`);
-        res.send(processId.toString());
-      }
-    } 
-    const body = await req.body;
-    const ownUrl = req.protocol + '://' + req.get('host') + req.originalUrl;
-    processStart(sendProcessId, body, ownUrl, configList);
-  });
 
-  app.listen(port, '0.0.0.0', () => {
+  app.listen(port, '0.0.0.0', () => {  // Start listening on port {port}
     console.log(`Example app listening on port ${port}`);
   });
 }

API/RequestHandlers.js

@@ -1,7 +1,7 @@
 import fs from "fs";
 import FormData from "form-data";
 import path from "path";
-import crypto from "crypto";
+import { appendUrl } from "../App/Utils.js";
 import {
     UpdateMetadata,
     PostMetadata,
@@ -24,12 +24,6 @@ import {
   getMinerResourceOutputExtension,
   getMinerFile,
 } from "../App/ConfigUnpacker.js";
-import { 
-    appendUrl,
-    removeFile,
-    removeFolder,
-} from "../App/Utils.js";
-import { AxiosError } from "axios";
 
 export const getForeignMiner = async (body, venvInitId) => {
     const shadowConfig = body.Config;
@@ -51,15 +45,14 @@ export const getForeignMiner = async (body, venvInitId) => {
         console.log("error when getting external miner: " + shadowMinerGetResult.data);
         throw "Error retrieving miner from external source: " + shadowMinerGetResult.data;
     }
-    if(shadowExtension != "py"){ // Don't need to install dependencies if it's not a python script
+    if(shadowExtension != "py"){ // Requirements are only installed for python scripts
         console.log("not a python miner, returning");
         return shadowConfig;
     }
     console.log("is a python miner!");
     const requirementsPath = path.join(shadowFolderPath, "requirements.txt");
     const requirementsGetResult = await GetFile(requirementsUrl, requirementsPath)
     if(requirementsGetResult.status !== 200){
-        // removeFolder(shadowFolderPath); // TODO: Should delete the folder if it reaches here and fails, but only deletes the contents. Probably some asynchronous stuff.
         throw "Error retrieving requirements from external source: " + requirementsGetResult.data;
     }
     return shadowConfig;
@@ -68,23 +61,15 @@ export const getForeignMiner = async (body, venvInitId) => {
 export const getResourceFromRepo = async (url, filePath) => {
     const result = await GetFile(url, filePath);
     return result;
-    // return {
-    //     response: result ? "File saved" : "Exception when writing downloaded file to Tmp folder.",
-    //     status: !!result,
-    // }
 }
 
 export const updateMetadata = async (body, resourceId, isDynamic) => {
+    if(!resourceId) return "Never received any messages, resulting in the miner to never send a resource, which means it has no resourceId"
     console.log(`Updating metadata on url: ${appendUrl([getBodyOutputHostInit(body), resourceId]).toString()} to set Dynamic to: ${isDynamic}`);
     const data = new FormData();
     data.append("Dynamic", isDynamic.toString());  // If it's a stream miner, it should be marked as dynamic
-
     const res = await UpdateMetadata(getBodyOutputHostInit(body), resourceId, data);
     return res;
-    // return {
-    //     response: res.data,
-    //     status: res.status == 200,
-    // };
 };
 
 export const sendMetadata = async (body, minerToRun, ownUrl, parents) => {
@@ -95,9 +80,6 @@ export const sendMetadata = async (body, minerToRun, ownUrl, parents) => {
         SourceId: getMinerId(minerToRun),
         SourceLabel: getMinerLabel(minerToRun),
     });
-
-    console.log("Trying to send metadata");
-
     const data = new FormData();
     data.append("Host", getBodyOutputHost(body));                         // mqtt.eclipseprojects.io
     data.append("StreamTopic", getBodyOutputTopic(body));                 // FilteredAlphabetStream 
@@ -121,7 +103,7 @@ export const updateResourceOnRepo = async (body, minerResult, resourceId) => {
     const fileStream = fs.createReadStream(minerResult);
     const data = new FormData();
     data.append("field-name", fileStream, { knownLength: fileSizeInBytes });
-
+    
     const res = await UpdateFile(getBodyOutputHost(body), resourceId, data);
     return {
         response: res.data,

API/Requests.js

@@ -1,8 +1,6 @@
 import axios from "axios";
 import fs from "fs";
 import { appendUrl } from "../App/Utils.js";
-// TODO: Delete this dict and its uses before hand-in
-var numUpdates = {};
 
 export const GetMetadata = async (path, resourceId) => {
   const url = appendUrl([path, resourceId]).toString();
@@ -45,26 +43,8 @@ export const PostMetadata = async (path, data) => {
     });;
   return {data: res.data, status: res.status};
 }
-// TODO: Delete? Don't think it was used
-// export const GetResource = async (path, resourceId) => {
-//   const url = appendUrl([path, resourceId]).toString();
-//   const res = await axios.get(url)
-//     .then((response) => {
-//       return {data: response.data, status: response.status};
-//     })
-//     .catch(error => {
-//       console.error("CATCH: fetch error: ");
-//       console.error(error);
-//       return {data: error.message, status: error.status};
-//     });
-//   return {data: res.data, status: res.status};
-// }
-export const UpdateFile = async (path, resourceId, data) => {
-  // TODO: Delete uses of "numUpdates" below before hand-in. Just to track how many updates it's got.
-  // if(numUpdates[resourceId] == null) numUpdates[resourceId] = 1;
-  // else numUpdates[resourceId] += 1;
-  // console.log(`Num updates for ${resourceId} = ${numUpdates[resourceId]}`);
 
+export const UpdateFile = async (path, resourceId, data) => {
   const url = appendUrl([path, resourceId]).toString();
   const res = await axios.put(url, data)
     .then((response) => {

App/ConfigUnpacker.js

@@ -10,11 +10,6 @@ const configName = "../config.json";
 const configPath = path.join(__dirname, configName);
 
 export function writeConfig(configList) {
-    // var configList = getConfig();
-    // if (configList.filter(e => e.MinerId === config.MinerId).length > 0) {
-    //     console.log("Config with that ID already exist. Leaving it as is. Overwriting shouldn't be done through here.");
-    //     return;
-    // }
     var configListString = JSON.stringify(configList, null, 2);
     fs.writeFileSync(configPath, configListString, 'utf8', function(err) {
         if(err) throw err;

App/ProcessHelper.js

@@ -1,3 +1,4 @@
+import spawn from "child_process";
 var processDict = {}; // Dict of all processes
 var processStatusDict = {}; // Dict of all process status objects
 
@@ -77,9 +78,20 @@ export function setProcess(processId, process) {
     processDict[processId] = process;
     console.log(`Process added to dict. All current process IDs: ${Object.keys(processDict)}`);
 }
+// export function killProcess(processId) {
+//     if(getProcess(processId)) {
+//         getProcess(processId).kill();
+//         deleteFromProcessDict(processId);
+//     }
+// }
 export function killProcess(processId) {
+
     if(getProcess(processId)) {
-        getProcess(processId).kill();
+        spawn.exec(`taskkill /PID ${processId} /F /T`, (stdout) => {
+            if(stdout) {
+              console.log(stdout);
+            }
+          });
         deleteFromProcessDict(processId);
     }
 }