fix: parse and push sources to db

Author: frankpagan

CoCreate.config.js

@@ -4,13 +4,13 @@ module.exports = {
         "organization_Id": "5ff747727005da1c272740ab",
         "host": "general.cocreate.app"
     },
-    
-    "sources": [{
+    "sources": [
+        {
             "entry": "./docs/index.html",
             "collection": "files",
             "document_id": "601460b19f64ba1680b8669b",
             "key": "src",
-            "data":{
+            "data": {
                 "name": "index.html",
                 "path": "/docs/docs/index.html",
                 "domains": [
@@ -22,21 +22,5 @@ module.exports = {
                 "website_id": "61381ed8829b690010a4f2b2"
             }
         }
-    ],
-
-	"extract": {
-		"directory": "./src/",
-		"extensions": [
-			"js",
-			"css",
-			"html"
-		],
-		"ignores": [
-			"node_modules",
-			"vendor",
-			"bower_components",
-			"archive"
-		]
-	}
-}
-
+    ]
+}

result.json

@@ -1 +1 @@
-[[{"collection":"docs","data":{"description":"Css comment extract example\n   testing....\n   end","extension":".css","file_path":"./test_files/box-shadow.css"},"metaData":"./test_files/box-shadow.css"}],[{"collection":"docs","data":{"description":"CoCreateAction's init function","extension":".js","file_path":"./test_files/CoCreate-action.js"},"metaData":"./test_files/CoCreate-action.js"}],[{"collection":"docs","data":{"description":"updateDocument({\n   collection: \"test123\",\n   document_id: \"document_id\",\n   data:{\n   \texample: “some example can be html json etc”,\n   \tdescription:  “update documnets if document does not exist otherwise create”\n   },\n })","extension":".js","file_path":"./test_files/CoCreate.js"},"metaData":"./test_files/CoCreate.js"},{"collection":"docs","data":{"description":"readDocument({\n   collection: \"test123\",\n   document_id: \"document_id\",\n   element: “xxxx”,\n   metaData: \"xxxx\",\n   exclude_fields: [] \n })","extension":".js","file_path":"./test_files/CoCreate.js"},"metaData":"./test_files/CoCreate.js"},{"collection":"docs","data":{"description":"deleteDocument({\n   namespace: '',\n   room: '',\n   broadcast: true/false,\n   broadcast_sender: true/false,\n   \n   collection: \"module\",\n   document_id: \"\",\n   element: “xxxx”,\n   metadata: \"xxxx\"\n })","extension":".js","file_path":"./test_files/CoCreate.js"},"metaData":"./test_files/CoCreate.js"}],[{"collection":"docs","data":{"description":"<script src=\"https://cdn.cocreate.app/latest/CoCreate.min.js\" ></script>\n\t\t\n\t\tParse <testing class=\"\"></testing>\n\t\ttesting...","extension":".html","file_path":"./test_files/test.html"},"metaData":"./test_files/test.html"}]]
+[]

src/index.1.js

@@ -0,0 +1,117 @@
+const CoCreateCrud = require('@cocreate/crud-client')
+const CoCreateSocket = require('@cocreate/socket-client')
+
+const CoCreateExtract = require('./extract')
+
+const fs = require('fs');
+const path = require('path');
+let config;
+
+let jsConfig = path.resolve(process.cwd(), 'CoCreate.config.js');
+let jsonConfig = path.resolve(process.cwd(), 'CoCreate.config.json')
+if (fs.existsSync(jsConfig))
+	config = require(jsConfig);
+else if (fs.existsSync(jsonConfig)) {
+	config = require(jsonConfig)
+}
+else {
+	process.exit()
+	console.log('config not found.')
+}
+
+const { crud, extract, sources, config : socketConfig } = config;
+
+/** init cocreatecrud and socket **/
+let socket = new CoCreateSocket("ws");
+CoCreateCrud.setSocket(socket);
+socket.create({
+	namespace: socketConfig.organization_Id,
+	room: null,
+	host: socketConfig.host
+})
+
+const commonParam = {
+	apiKey : socketConfig.apiKey,
+	organization_id : socketConfig.organization_Id,
+	broadcast: false
+}
+
+async function runStore (info, type) {
+	try {
+		let status = false;
+		const event = "docEvent" + Date.now()
+		if (!info.document_id) {
+			status = CoCreateCrud.createDocument({
+				...commonParam,
+				...info,
+				event
+			})
+		} else {
+			status = CoCreateCrud.updateDocument({
+				...commonParam,
+				...info,
+				upsert: true,
+				event
+			})
+		}
+		if (status) {
+			
+			let response = await CoCreateCrud.listenAsync(event)
+			console.log('type ------------------------- ', type)
+			console.log(response)
+		}
+	} catch (err) {
+		console.log(err);
+	}
+} 
+
+/**
+ * Extract comments and store into db
+ */
+if (extract) {
+	let result = CoCreateExtract(extract.directory, extract.ignores, extract.extensions);
+	fs.writeFileSync('result.json', JSON.stringify(result), 'utf8')
+	
+	result.forEach((docs) => {
+		docs.forEach(async(doc) => {
+			if (!doc.collection) return;
+			await runStore(doc, 'extract')
+		})
+	})
+}
+
+/**
+ * update and create document by config crud
+ */
+
+// if (crud) {
+// 	crud.forEach(async (info) => {
+// 		await runStore(info, 'crud')
+// 	})
+// }
+
+/**
+ * Store html files by config sources
+ **/
+if (sources) {
+	sources.forEach(async ({entry, collection, document_id, key, data}) => {
+		if (!path) return;
+
+		let content = fs.readFileSync(path, 'utf8');
+
+		if (content && key && collection) {
+			if (!data) data = {};
+			let storeData = {
+				[key]: content,
+				...data
+			};
+			await runStore({collection, document_id, data: storeData}, 'sources');
+		}
+	})
+}
+
+console.log('end....')
+
+setTimeout(function(){
+	process.exit()
+}, 1000 * 30)

src/index.js

@@ -1,25 +1,28 @@
 const CoCreateCrud = require('@cocreate/crud-client')
 const CoCreateSocket = require('@cocreate/socket-client')
-
+const mime = require('mime-types')
 const CoCreateExtract = require('./extract')
 
 const fs = require('fs');
 const path = require('path');
 let config;
 
 let jsConfig = path.resolve(process.cwd(), 'CoCreate.config.js');
-let jsonConfig = path.resolve(process.cwd(), 'CoCreate.config.json')
+// let jsonConfig = path.resolve(process.cwd(), 'CoCreate.config.json')
 if (fs.existsSync(jsConfig))
 	config = require(jsConfig);
-else if (fs.existsSync(jsonConfig)) {
-	config = require(jsonConfig)
-}
+// else if (fs.existsSync(jsonConfig)) {
+// 	config = require(jsonConfig)
+// }
 else {
 	process.exit()
 	console.log('config not found.')
 }
 
 const { crud, extract, sources, config : socketConfig } = config;
+console.log(config)
+
+process.env['NODE_TLS_REJECT_UNAUTHORIZED'] = 0
 
 /** init cocreatecrud and socket **/
 let socket = new CoCreateSocket("ws");
@@ -55,16 +58,92 @@ async function runStore (info, type) {
 			})
 		}
 		if (status) {
-			
 			let response = await CoCreateCrud.listenAsync(event)
 			console.log('type ------------------------- ', type)
 			console.log(response)
+			return response;
 		}
 	} catch (err) {
 		console.log(err);
+		return null;
 	}
 } 
 
+/**
+ * update and create document by config crud
+ */
+
+if (crud) {
+	crud.forEach(async (info) => {
+		await runStore(info, 'crud')
+	})
+}
+
+/**
+ * Store html files by config sources
+ **/
+if (sources) {
+	let new_sources_list = [];
+
+	async function runSources() {
+		for (let i = 0; i < sources.length; i++) {
+			const { entry, collection, document_id, key, data } = sources[i];
+			
+			let new_source = {...sources[i]};
+			let response = {};
+			if (entry) {
+				
+				try {
+					let read_type = 'utf8'
+					let mime_type = mime.lookup(entry) || 'text/html';
+					if (/^(image|audio|video)\/[-+.\w]+/.test(mime_type)) {
+						read_type = 'base64'
+					}
+					
+					let binary = fs.readFileSync(entry);
+					
+					let content = new Buffer(binary).toString(read_type);
+
+					if (content && key && collection) {
+						if (!data) data = {};
+						let storeData = {
+							[key]: content,
+							...data,
+						};
+						
+						response = await runStore({collection, document_id, data: storeData}, 'sources');
+					}
+				} catch (err) {
+					console.log(err)
+				}
+				if (response.document_id) {
+					new_source.document_id = response.document_id
+				}
+			}
+			new_sources_list.push(new_source)
+		}
+		return new_sources_list
+
+	}
+	
+	runSources().then((data) => {
+		
+		console.log(data)
+		let new_config = {
+			config: socketConfig,
+			sources: new_sources_list,
+			crud: crud,
+		}
+		
+		let write_str = JSON.stringify(new_config, null, 4)
+		write_str = "module.exports = " + write_str;
+
+		fs.writeFileSync(jsConfig, write_str);
+		// fs.writeFileSync(jsonConfig, write_str);
+
+	})
+}
+
 /**
  * Extract comments and store into db
  */
@@ -93,25 +172,14 @@ if (extract) {
 /**
  * Store html files by config sources
  **/
-if (sources) {
-	sources.forEach(async ({entry, collection, document_id, key, data}) => {
-		if (!path) return;
-
-		let content = fs.readFileSync(path, 'utf8');
-
-		if (content && key && collection) {
-			if (!data) data = {};
-			let storeData = {
-				[key]: content,
-				...data
-			};
-			await runStore({collection, document_id, data: storeData}, 'sources');
-		}
-	})
-}
+
 
 console.log('end....')
 
 setTimeout(function(){
 	process.exit()
-}, 1000 * 30)
+}, 1000 * 60)
+
+
+
+