Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .github/workflows/e2e-bm.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,7 @@ jobs:

- name: Install Flame with flmadm and systemd
run: |
sudo ./target/release/flmadm install --src-dir . --skip-build --prefix $INSTALL_PREFIX --enable
sudo ./target/release/flmadm install --all --src-dir . --skip-build --prefix $INSTALL_PREFIX --enable
echo "$INSTALL_PREFIX/bin" >> $GITHUB_PATH

- name: Setup local configuration and packages directory
Expand Down
12 changes: 11 additions & 1 deletion Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

15 changes: 8 additions & 7 deletions common/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -181,12 +181,13 @@ pub fn default_applications() -> HashMap<String, ApplicationAttributes> {
"description": "The output of the script in UTF-8."
});

// Get FLAME_HOME from environment or use default
let flame_home = std::env::var("FLAME_HOME").unwrap_or_else(|_| "/usr/local/flame".to_string());
let flmexec_cmd = format!("{}/bin/flmexec-service", flame_home);
let flmping_cmd = format!("{}/bin/flmping-service", flame_home);
let flmping_url = format!("file://{}/bin/flmping-service", flame_home);
let flamepy_sdk_path = format!("file://{}/sdk/python", flame_home);
// Use ${FLAME_HOME} variable substitution syntax
// This will be expanded at runtime by the executor to the actual FLAME_HOME path
let flmexec_cmd = "${FLAME_HOME}/bin/flmexec-service".to_string();
let flmping_cmd = "${FLAME_HOME}/bin/flmping-service".to_string();
let uv_cmd = "${FLAME_HOME}/bin/uv".to_string();
let flmping_url = "file://${FLAME_HOME}/bin/flmping-service".to_string();
let flamepy_sdk_path = "file://${FLAME_HOME}/sdk/python".to_string();

HashMap::from([
(
Expand Down Expand Up @@ -222,7 +223,7 @@ pub fn default_applications() -> HashMap<String, ApplicationAttributes> {
"The Flame Runner application for executing customized Python applications."
.to_string(),
),
command: Some("/usr/bin/uv".to_string()),
command: Some(uv_cmd),
arguments: vec![
"run".to_string(),
"--with".to_string(),
Expand Down
19 changes: 0 additions & 19 deletions docker/Dockerfile.cache

This file was deleted.

37 changes: 24 additions & 13 deletions docker/Dockerfile.console
Original file line number Diff line number Diff line change
Expand Up @@ -2,25 +2,36 @@ FROM rust:1.88 AS builder

WORKDIR /usr/src/flame
COPY . .
RUN apt-get update && apt-get install -y protobuf-compiler pkg-config libssl-dev
RUN cargo install --path ./flmctl
RUN cargo install --path ./flmping
RUN cargo install --path ./flmexec

FROM ubuntu:24.04
RUN apt-get update && apt-get install -y protobuf-compiler pkg-config libssl-dev
RUN cargo build --release

RUN apt-get update && apt-get install -y wget vim iputils-ping ssh curl python3 python3-pip
COPY --from=builder /usr/local/cargo/bin/flmping /usr/local/bin/flmping
COPY --from=builder /usr/local/cargo/bin/flmctl /usr/local/bin/flmctl
COPY --from=builder /usr/local/cargo/bin/flmexec /usr/local/bin/flmexec
# Build flmadm for installation
RUN cargo install --path ./flmadm

# Copy uv before flmadm install (flmadm will detect and copy it to FLAME_HOME/bin)
COPY --from=ghcr.io/astral-sh/uv:0.9.26 /uv /uvx /usr/bin/

RUN chmod +x /usr/local/bin/*
# Install using flmadm with client profile
RUN /usr/local/cargo/bin/flmadm install \
--src-dir /usr/src/flame \
--prefix /usr/local/flame \
--client \
--skip-build \
--force

FROM ubuntu:24.04

RUN apt-get update && apt-get install -y wget vim iputils-ping ssh curl python3 python3-pip && rm -rf /var/lib/apt/lists/*

# Copy the entire installation from builder (including uv in bin/)
COPY --from=builder /usr/local/flame /usr/local/flame

COPY ./sdk/python /usr/local/flame/sdk/python
# Add flame binaries to PATH (includes uv)
ENV PATH="/usr/local/flame/bin:${PATH}"
ENV FLAME_HOME=/usr/local/flame

# Install flamepy and pytest for test client using uv
RUN uv pip install --system --break-system-packages --no-cache /usr/local/flame/sdk/python pytest
# Install flamepy and pytest for test client using uv from FLAME_HOME
RUN /usr/local/flame/bin/uv pip install --system --break-system-packages --no-cache /usr/local/flame/sdk/python pytest

CMD ["service", "ssh", "start", "-D"]
32 changes: 18 additions & 14 deletions docker/Dockerfile.fem
Original file line number Diff line number Diff line change
Expand Up @@ -4,27 +4,31 @@ WORKDIR /usr/src/flame
COPY . .

RUN apt-get update && apt-get install -y protobuf-compiler pkg-config libssl-dev
RUN cargo install --path ./executor_manager
RUN cargo install --path ./flmping
RUN cargo install --path ./flmexec
RUN cargo build --release

FROM ubuntu:24.04

RUN apt-get update && apt-get install -y python3-pip
# Build flmadm for installation
RUN cargo install --path ./flmadm

RUN mkdir -p /usr/local/flame/bin /usr/local/flame/work/tmp /usr/local/flame/sdk
# Copy uv before flmadm install (flmadm will detect and copy it to FLAME_HOME/bin)
COPY --from=ghcr.io/astral-sh/uv:0.9.26 /uv /uvx /usr/bin/

WORKDIR /usr/local/flame/work
# Install using flmadm with worker profile
RUN /usr/local/cargo/bin/flmadm install \
--src-dir /usr/src/flame \
--prefix /usr/local/flame \
--worker \
--no-systemd \
--skip-build \
--force

COPY --from=ghcr.io/astral-sh/uv:0.9.26 /uv /uvx /usr/bin/
FROM ubuntu:24.04

COPY --from=builder /usr/local/cargo/bin/flame-executor-manager /usr/local/flame/bin/flame-executor-manager
COPY --from=builder /usr/local/cargo/bin/flmping-service /usr/local/flame/bin/flmping-service
COPY --from=builder /usr/local/cargo/bin/flmexec-service /usr/local/flame/bin/flmexec-service
RUN apt-get update && apt-get install -y python3-pip && rm -rf /var/lib/apt/lists/*

RUN chmod +x /usr/local/flame/bin/*
WORKDIR /usr/local/flame/work

COPY ./sdk/python /usr/local/flame/sdk/python
# Copy the entire installation from builder (including uv in bin/)
COPY --from=builder /usr/local/flame /usr/local/flame

ENV FLAME_HOME=/usr/local/flame

Expand Down
23 changes: 15 additions & 8 deletions docker/Dockerfile.fsm
Original file line number Diff line number Diff line change
Expand Up @@ -4,19 +4,26 @@ WORKDIR /usr/src/flame
COPY . .

RUN apt-get update && apt-get install -y protobuf-compiler pkg-config libssl-dev
RUN cargo install --path ./session_manager
RUN cargo build --release

# Build flmadm for installation
RUN cargo install --path ./flmadm

# Install using flmadm with control-plane profile
RUN /usr/local/cargo/bin/flmadm install \
--src-dir /usr/src/flame \
--prefix /usr/local/flame \
--control-plane \
--no-systemd \
--skip-build \
--force

FROM ubuntu:24.04

RUN mkdir -p /usr/local/flame/bin
RUN mkdir -p /usr/local/flame/work
RUN mkdir -p /usr/local/flame/migrations
WORKDIR /usr/local/flame

COPY session_manager/migrations /usr/local/flame/migrations
COPY --from=builder /usr/local/cargo/bin/flame-session-manager /usr/local/flame/bin/flame-session-manager

RUN chmod +x /usr/local/flame/bin/*
# Copy the entire installation from builder
COPY --from=builder /usr/local/flame /usr/local/flame

ENV FLAME_HOME=/usr/local/flame

Expand Down
2 changes: 1 addition & 1 deletion docs/designs/RFE318-cache/STATUS.md
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ The object cache is now provided as an embedded library within the `flame-execut
- ✅ In-memory index with HashMap
- ✅ Object loading from disk on startup
- ✅ Configuration support (flame-cluster.yaml with storage path)
- ✅ Docker integration (Dockerfile.cache, compose.yaml, Makefile)
- ✅ Docker integration (embedded in executor-manager, compose.yaml, Makefile)

### API Operations
- ✅ `get_flight_info`: Returns flight metadata for objects
Expand Down
2 changes: 1 addition & 1 deletion e2e/tests/test_agent.py
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ def setup_test_env():
FLM_TEST_APP,
flamepy.ApplicationAttributes(
shim=flamepy.Shim.Host,
command="uv",
command="${FLAME_HOME}/bin/uv",
working_directory="/opt/e2e",
environments={"FLAME_LOG_LEVEL": "DEBUG"},
arguments=["run", "src/e2e/instance_svc.py", "src/e2e/api.py"],
Expand Down
10 changes: 6 additions & 4 deletions e2e/tests/test_core.py
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ def setup_test_env():
FLM_TEST_SVC_APP,
flamepy.ApplicationAttributes(
shim=flamepy.Shim.Host,
command="uv",
command="${FLAME_HOME}/bin/uv",
working_directory="/opt/e2e",
environments={"FLAME_LOG_LEVEL": "DEBUG"},
arguments=["run", "src/e2e/basic_svc.py", "src/e2e/api.py"],
Expand Down Expand Up @@ -137,7 +137,8 @@ def test_application_context_info():
"Host" in response.application_context.shim
or "host" in response.application_context.shim.lower()
)
assert response.application_context.command == "uv"
# Command should use FLAME_HOME environment variable
assert response.application_context.command == "${FLAME_HOME}/bin/uv"
assert response.application_context.working_directory == "/opt/e2e"
assert response.application_context.url == FLM_TEST_SVC_APP_URL

Expand Down Expand Up @@ -177,7 +178,8 @@ def test_all_context_info():

# Check application context details
assert response.application_context.name == FLM_TEST_SVC_APP
assert response.application_context.command == "uv"
# Command should use FLAME_HOME environment variable
assert response.application_context.command == "${FLAME_HOME}/bin/uv"
assert response.application_context.working_directory == "/opt/e2e"
assert response.application_context.url == FLM_TEST_SVC_APP_URL

Expand Down Expand Up @@ -373,7 +375,7 @@ def test_task_invoke_exception_handling():
FLM_ERROR_SVC_APP,
flamepy.ApplicationAttributes(
shim=flamepy.Shim.Host,
command="uv",
command="${FLAME_HOME}/bin/uv",
working_directory="/opt/e2e",
environments={"FLAME_LOG_LEVEL": "DEBUG"},
arguments=["run", "src/e2e/error_svc.py"],
Expand Down
3 changes: 2 additions & 1 deletion e2e/tests/test_flmrun.py
Original file line number Diff line number Diff line change
Expand Up @@ -79,7 +79,8 @@ def test_flmrun_application_registered():
assert flmrun.name == FLMRUN_E2E_APP
assert flmrun.shim == flamepy.Shim.Host
assert flmrun.state == flamepy.ApplicationState.ENABLED
assert flmrun.command == "/usr/bin/uv"
# Command should use FLAME_HOME environment variable
assert flmrun.command == "${FLAME_HOME}/bin/uv"


def test_flmrun_sum_function():
Expand Down
1 change: 1 addition & 0 deletions executor_manager/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@ wasmtime = "16"
wasmtime-wasi = "16"
anyhow = "1"
tokio-stream = { workspace = true }
shellexpand = "3.1"

# Dependencies for embedded object cache
arrow = "53"
Expand Down
19 changes: 18 additions & 1 deletion executor_manager/src/shims/host_shim.rs
Original file line number Diff line number Diff line change
Expand Up @@ -197,15 +197,32 @@ impl HostShim {
Ok(envs)
}

/// Expand environment variables in a string
/// Supports both ${VAR} and $VAR syntax
fn expand_env_vars(s: &str) -> String {
shellexpand::env(s)
.unwrap_or(std::borrow::Cow::Borrowed(s))
.into_owned()
}

fn launch_instance(
app: &ApplicationContext,
executor: &Executor,
endpoint: &str,
) -> Result<HostInstance, FlameError> {
trace_fn!("HostShim::launch_instance");

// Expand environment variables in command and arguments
let command = app.command.clone().unwrap_or_default();
let args = app.arguments.clone();
let command = Self::expand_env_vars(&command);

let args: Vec<String> = app
.arguments
.clone()
.iter()
.map(|arg| Self::expand_env_vars(arg))
.collect();

let log_level = env::var(RUST_LOG).unwrap_or(String::from(DEFAULT_SVC_LOG_LEVEL));

let mut envs = app.environments.clone();
Expand Down
Loading
Loading