diff --git a/docker/named_pipe/listen.sh b/docker/named_pipe/listen.sh index d281ae7..8f70d43 100755 --- a/docker/named_pipe/listen.sh +++ b/docker/named_pipe/listen.sh @@ -1,3 +1,21 @@ #!/bin/sh +while true; do + raw=$(cat pipe) + + if echo "$raw" | grep -q "^RESPOND::"; then + cmd="${raw##RESPOND::}" + response_file=$(mktemp) + eval "$cmd" > "$response_file" 2>&1 + ( + while ! mkdir output_pipe.lock 2>/dev/null; do + sleep 1 + done -while true; do eval "$(cat pipe)"; done + cat "$response_file" > output_pipe + rm -f "$response_file" + rmdir output_pipe.lock + ) & + else + eval "$raw" + fi +done \ No newline at end of file diff --git a/docs/admin/README.md b/docs/admin/README.md index 046e45d..3a0d3a2 100644 --- a/docs/admin/README.md +++ b/docs/admin/README.md @@ -47,6 +47,7 @@ docker compose up --build -d ### 4. Setup Named Pipes Create a pipe in the `docker/named_pipe` directory by executing `mkfifo docker/named_pipe/pipe`. +Create another one by executing `mkfifo docker/named_pipe/output_pipe`. Navigate to the `docker/named_pipe` directory and execute the `listen.sh` script to allow the application to run commands on the host. ```bash cd docker/named_pipe diff --git a/src/backend/main.ts b/src/backend/main.ts index bbace42..726c03e 100644 --- a/src/backend/main.ts +++ b/src/backend/main.ts @@ -3,7 +3,7 @@ import { addScript, deleteScript } from "./scripts.ts"; import { checkJWT } from "./utils/jwt.ts"; import { addMaps, deleteMaps, getMaps, getDeploymentsByRepo, getUserToken } from "./db.ts"; import { encryptEnv, decryptEnv } from "./utils/crypto.ts"; - +import { canAllocateStorage } from "./utils/container-storage.ts"; // ... skipping to githubWebhook @@ -57,6 +57,19 @@ async function addSubdomain(ctx: Context) { } // We keep deployment config (port, stack, etc.) in the document to store them in DB for webhook usage + if(copy.volume_needed=="Yes"){const storageCheck=await canAllocateStorage(100); + if(!storageCheck.can_allocate){ + ctx.response.status=400; + ctx.response.body = { + status: "failed", + error: "INSUFFICIENT_STORAGE", + message: storageCheck.reason || "Not enough disk space", + available_mb: storageCheck.available_mb, + requested_mb: storageCheck.requested_mb, + }; + console.log(storageCheck.available_mb); + return; + }} const success: boolean = await addMaps(document); @@ -132,6 +145,7 @@ async function addSubdomain(ctx: Context) { copy.env_content, copy.static_content, copy.dockerfile_present, + copy.volume_needed, copy.stack, copy.port, copy.build_cmds, diff --git a/src/backend/scripts.ts b/src/backend/scripts.ts index a0108c6..8d88b4c 100644 --- a/src/backend/scripts.ts +++ b/src/backend/scripts.ts @@ -32,6 +32,7 @@ async function addScript( env_content: string, static_content: string, dockerfile_present: string, + volume_needed:string, stack: string, port: string, build_cmds: string, @@ -40,7 +41,9 @@ async function addScript( const resource = shellEscape(document.resource, "resource"); const safePort = shellEscape(port, "port"); const memLimit = shellEscape(MEMORY_LIMIT || "512m", "MEMORY_LIMIT"); - + volume_needed=(volume_needed=="Yes").toString(); + const volumeNeeded=shellEscape(volume_needed,"false"); + console.log(`volume needed is ${volumeNeeded}`); if (document.resource_type === "URL") { await safeExec( `bash -c "echo 'bash ../../src/backend/shell_scripts/automate.sh -u ${resource} ${subdomain}' > /hostpipe/pipe"`, @@ -52,7 +55,7 @@ async function addScript( } else if (document.resource_type === "GITHUB" && static_content == "Yes") { await Deno.writeTextFile(`/hostpipe/.env`, env_content || ""); await safeExec( - `bash -c "echo 'bash ../../src/backend/shell_scripts/container.sh -s ${subdomain} ${resource} 80 ${memLimit}' > /hostpipe/pipe"`, + `bash -c "echo 'bash ../../src/backend/shell_scripts/container.sh -s ${subdomain} ${resource} 80 ${memLimit} ${volumeNeeded}' > /hostpipe/pipe"`, ); } else if (document.resource_type === "GITHUB" && static_content == "No") { if (dockerfile_present === 'No') { @@ -60,11 +63,11 @@ async function addScript( await Deno.writeTextFile(`/hostpipe/.dockerignore`, dockerignore(stack || "")); await Deno.writeTextFile(`/hostpipe/.env`, env_content || ""); await safeExec( - `bash -c "echo 'bash ../../src/backend/shell_scripts/container.sh -g ${subdomain} ${resource} ${safePort} ${memLimit}' > /hostpipe/pipe"`, + `bash -c "echo 'bash ../../src/backend/shell_scripts/container.sh -g ${subdomain} ${resource} ${safePort} ${memLimit} ${volumeNeeded}' > /hostpipe/pipe"`, ); } else if (dockerfile_present === 'Yes') { await safeExec( - `bash -c "echo 'bash ../../src/backend/shell_scripts/container.sh -d ${subdomain} ${resource} ${safePort} ${memLimit}' > /hostpipe/pipe"`, + `bash -c "echo 'bash ../../src/backend/shell_scripts/container.sh -d ${subdomain} ${resource} ${safePort} ${memLimit} ${volumeNeeded}' > /hostpipe/pipe"`, ); } } diff --git a/src/backend/shell_scripts/container.sh b/src/backend/shell_scripts/container.sh index e0efd50..c030427 100755 --- a/src/backend/shell_scripts/container.sh +++ b/src/backend/shell_scripts/container.sh @@ -5,9 +5,15 @@ name=$2 resource=$3 exp_port=$4 max_mem=$5 +enable_volume=$6 available_ports=() - +STORAGE_ROOT="/mnt/storage" +PROJECT_STORAGE="$STORAGE_ROOT/$name" +PROJECT_IMG="$STORAGE_ROOT/$name.img" +SIZE_MB=100 +sudo mkdir -p "$STORAGE_ROOT" +sudo chmod 755 "$STORAGE_ROOT" for ((port=PORT_MIN; port<=PORT_MAX; port++)); do if ! ss -ln src :$port | grep -q "\<$port\>"; then available_ports+=($port) @@ -17,9 +23,9 @@ done echo "Available ports: ${available_ports[56]}" AVAILABLE=0 echo "Creating subdomain $name" -git clone $resource $name -sudo cp .env $name/ -cd $name +git clone $resource "$name" +sudo cp .env "$name/" +cd "$name" if [ $flag = "-g" ]; then sudo cp ../Dockerfile ./ @@ -30,19 +36,47 @@ elif [ $flag = "-s" ]; then COPY . /usr/share/nginx/html " > Dockerfile fi +if [ "$enable_volume" = "true" ]; then + echo "Creating persistent storage at $PROJECT_STORAGE" + if [ ! -f "$PROJECT_IMG" ]; then + sudo dd if=/dev/zero of="$PROJECT_IMG" bs=1M count=$SIZE_MB + sudo mkfs.ext4 "$PROJECT_IMG" + fi + sudo mkdir -p "$PROJECT_STORAGE" + if ! mountpoint -q "$PROJECT_STORAGE"; then + sudo mount -o loop "$PROJECT_IMG" "$PROJECT_STORAGE" + fi + sudo chmod 777 "$PROJECT_STORAGE" +fi sudo docker build -t $name . # Safety net: If the frontend sends double requests from spam-clicking, forcefully remove any zombie container holding the name sudo docker rm -f $name 2>/dev/null || true -sudo docker run --memory=$max_mem --name=$name -d -p ${available_ports[$AVAILABLE]}:$exp_port $name +if [ "$enable_volume" = "true" ]; then + sudo docker run \ + --memory=$max_mem \ + --name="$name" \ + -d \ + -p ${available_ports[$AVAILABLE]}:$exp_port \ + -v "$PROJECT_STORAGE":/app/data \ + -e DATA_DIR=/app/data \ + "$name" +else + sudo docker run \ + --memory=$max_mem \ + --name="$name" \ + -d \ + -p ${available_ports[$AVAILABLE]}:$exp_port \ + "$name" +fi cd .. -sudo rm -rf $name +sudo rm -rf "$name" sudo rm Dockerfile sudo rm .env -sudo touch /etc/nginx/sites-available/$name.conf -sudo chmod 666 /etc/nginx/sites-available/$name.conf +sudo touch "/etc/nginx/sites-available/$name".conf +sudo chmod 666 "/etc/nginx/sites-available/$name.conf" sudo echo "# Virtual Host configuration for $name server { listen 80; @@ -58,6 +92,6 @@ sudo echo "# Virtual Host configuration for $name } charset utf-8; client_max_body_size 20M; - }" > /etc/nginx/sites-available/$name.conf -sudo ln -s /etc/nginx/sites-available/$name.conf /etc/nginx/sites-enabled/$name.conf + }" > "/etc/nginx/sites-available/$name.conf" +sudo ln -s "/etc/nginx/sites-available/$name.conf" "/etc/nginx/sites-enabled/$name.conf" sudo systemctl reload nginx diff --git a/src/backend/shell_scripts/delete.sh b/src/backend/shell_scripts/delete.sh index 30c80c6..f75a51c 100644 --- a/src/backend/shell_scripts/delete.sh +++ b/src/backend/shell_scripts/delete.sh @@ -6,9 +6,13 @@ id -u # Assign the arguments to variables -arg1=$1 +name=$1 -echo "Deleting... $arg1" +echo "Deleting... $name" + +STORAGE_ROOT="/mnt/storage" +PROJECT_STORAGE="$STORAGE_ROOT/$name" +PROJECT_IMG="$STORAGE_ROOT/$name.img" sudo rm /etc/nginx/sites-available/$1.conf sudo rm /etc/nginx/sites-enabled/$1.conf @@ -16,4 +20,18 @@ sudo docker stop $1 sudo docker rm $1 sudo docker rmi $1 +if mountpoint -q "$PROJECT_STORAGE"; then + echo "Unmounting volume..." + sudo umount "$PROJECT_STORAGE" +fi + +if [ -d "$PROJECT_STORAGE" ]; then + sudo rm -rf "$PROJECT_STORAGE" +fi + +if [ -f "$PROJECT_IMG" ]; then + echo "Deleting volume image..." + sudo rm -f "$PROJECT_IMG" +fi + sudo systemctl reload nginx diff --git a/src/backend/utils/container-storage.ts b/src/backend/utils/container-storage.ts new file mode 100644 index 0000000..17259e1 --- /dev/null +++ b/src/backend/utils/container-storage.ts @@ -0,0 +1,63 @@ +import { exec } from "../dependencies.ts"; + +const STORAGE_TIMEOUT_MS = 5000; + +let storagePipeLock: Promise = Promise.resolve(); +async function withStoragePipeLock(operation: () => Promise): Promise { + const previousLock = storagePipeLock; + let releaseLock!: () => void; + storagePipeLock = new Promise((resolve) => { + releaseLock = resolve; + }); + await previousLock; + try { + return await operation(); + } finally { + releaseLock(); + } +} +export async function canAllocateStorage(requestedMb: number) { + const STORAGE_PATH = "/mnt/storage"; + const SAFETY_BUFFER_MB = 200; // keep buffer for system + docker + + try { + return await withStoragePipeLock(async () => { + const abortController = new AbortController(); + const timeoutId = setTimeout(() => abortController.abort(), STORAGE_TIMEOUT_MS); + try { + const responseProcess = new Deno.Command("sh", { + args: ["-c", "cat /hostpipe/output_pipe"], + }).output(); // don't await yet, just start it + await exec(`bash -c "echo 'RESPOND::df ${STORAGE_PATH} --output=avail' > /hostpipe/pipe"`); + const response = await responseProcess; + const output = new TextDecoder().decode(response.stdout).trim().split("\n"); + const availableKb = parseInt(output[1].trim()); + if (isNaN(availableKb)) { + throw new Error(`Unexpected df output: ${output}`); + } + const availableMb = Math.floor(availableKb / 1024); + const usableMb = availableMb - SAFETY_BUFFER_MB; + const canAllocate = usableMb >= requestedMb; + console.log(`can allocate ${canAllocate} memory`); + console.log(`Available memory is ${availableMb} requested is ${requestedMb}`); + return { + can_allocate: canAllocate, + available_mb: usableMb, + requested_mb: requestedMb, + reason: canAllocate ? null : "Not enough disk space", + }; + } finally { + clearTimeout(timeoutId); + } + }); + + } catch (err) { + console.log(`Error during memory check volume`); + return { + can_allocate: false, + available_mb: 0, + requested_mb: requestedMb, + reason: "Failed to check disk space", + }; + } +} \ No newline at end of file diff --git a/src/cli/features/createDomain.ts b/src/cli/features/createDomain.ts index d989ce7..0a1b020 100644 --- a/src/cli/features/createDomain.ts +++ b/src/cli/features/createDomain.ts @@ -85,6 +85,7 @@ export async function createDomain(userApiKey: string, user: string, provider: s if (response.data.status === 'success') { console.log(`✅ Domain '${subdomain}.${domain}' created successfully!`); } else { + if(response.data.error=="INSUFFICIENT_STORAGE")console.log("INSUFFICIENT_STORAGE") console.log('❌ Domain creation failed!'); console.log("Either the domain exist or the domain is not created"); } diff --git a/src/frontend/src/components/modal.vue b/src/frontend/src/components/modal.vue index ce13fc9..27e7e82 100644 --- a/src/frontend/src/components/modal.vue +++ b/src/frontend/src/components/modal.vue @@ -26,8 +26,8 @@ const domain = import.meta.env.VITE_APP_DOMAIN

- Yes - No + Yes + No

Stack:

@@ -35,6 +35,11 @@ const domain = import.meta.env.VITE_APP_DOMAIN
+
+
+ Yes + No +

Port:

Build Commands:

@@ -67,6 +72,7 @@ export default { env_content: 'key1 = value1', // Default prompt text static_content: 'No', dockerfile_present :'No', + volume_needed: 'No', port: '', stack: '', build_cmds: '', @@ -78,14 +84,15 @@ export default { methods: { submitForm() { console.log(this.subdomain, this.resource_type, this.resource); - create(this.subdomain, this.resource_type, this.resource, this.env_content, this.static_content,this.dockerfile_present,this.port, this.stack, this.build_cmds, this.enable_ci) + create(this.subdomain, this.resource_type, this.resource, this.env_content, this.static_content,this.dockerfile_present, this.volume_needed, this.port, this.stack, this.build_cmds,this.enable_ci) .then((res) => { console.log(res); if (res === 'Submitted') { this.closeModalAndReload(); } else { this.closeModal(); - alert('Failed to create subdomain'); + if(res=="insufficient_storage")alert("Insufficient storage to mount a volume"); + else alert('Failed to create subdomain'); setTimeout(() => { window.location.reload(); }, 1000); diff --git a/src/frontend/src/utils/create.ts b/src/frontend/src/utils/create.ts index e63ca70..0ef9f07 100644 --- a/src/frontend/src/utils/create.ts +++ b/src/frontend/src/utils/create.ts @@ -61,6 +61,7 @@ export async function create( env_content: string, static_content: string, dockerfile_present:string, + volume_needed:string, port: string, stack: string, build_cmds: string, @@ -89,6 +90,7 @@ export async function create( "env_content": env_content, "static_content": static_content, "dockerfile_present":dockerfile_present, + "volume_needed":volume_needed, "port": port, "build_cmds": build_cmds, "stack": stack, @@ -106,8 +108,11 @@ export async function create( body: JSON.stringify(body), }); const data = await resp.json(); - if (data.status === "failed") { - return "Failed"; + if(data.error=="INSUFFICIENT_STORAGE"){ + return "insufficient_storage"; + } + else if (data.status === "failed") { + return "failed"; } return "Submitted"; }