From ff6a064f9810052dce04bebe40743e3561b2c376 Mon Sep 17 00:00:00 2001 From: DerThorsten Date: Thu, 10 Feb 2022 10:32:29 +0100 Subject: [PATCH 01/24] hacky emscripten experiments --- boa/core/run_build.py | 5 +- boa/core/solver.py | 1 + .../emscripten_emscripten32/recipe.yaml | 31 + .../xtensor/conda_build_config.yaml | 774 ++++++++++++++++++ .../emscripten_emscripten-32/activate.sh | 6 + .../emscripten_emscripten-32/build.sh | 9 + .../emscripten_emscripten-32/deactivate.sh | 1 + .../emscripten_emscripten-32/recipe.yaml | 32 + tests/recipes-v2_2/xtensor/bld.bat | 8 + tests/recipes-v2_2/xtensor/build.sh | 4 + tests/recipes-v2_2/xtensor/recipe.yaml | 62 ++ 11 files changed, 932 insertions(+), 1 deletion(-) create mode 100644 tests/recipes-v2/emscripten_emscripten32/recipe.yaml create mode 100644 tests/recipes-v2/xtensor/conda_build_config.yaml create mode 100644 tests/recipes-v2_2/emscripten_emscripten-32/activate.sh create mode 100644 tests/recipes-v2_2/emscripten_emscripten-32/build.sh create mode 100644 tests/recipes-v2_2/emscripten_emscripten-32/deactivate.sh create mode 100644 tests/recipes-v2_2/emscripten_emscripten-32/recipe.yaml create mode 100644 tests/recipes-v2_2/xtensor/bld.bat create mode 100644 tests/recipes-v2_2/xtensor/build.sh create mode 100644 tests/recipes-v2_2/xtensor/recipe.yaml diff --git a/boa/core/run_build.py b/boa/core/run_build.py index 70193a75..ec79822f 100644 --- a/boa/core/run_build.py +++ b/boa/core/run_build.py @@ -405,7 +405,7 @@ def build_recipe( continue_on_failure: bool = False, rerun_build: bool = False, ): - + print("in build_recipe") ydoc = render(recipe_path, config=config) # We need to assemble the variants for each output variants = {} @@ -608,6 +608,7 @@ def build_recipe( elif isinstance(e, BoaRunBuildException): raise e else: + raise e exit(1) for o in sorted_outputs: @@ -654,6 +655,7 @@ def run_build(args): console.print(f"Updating build index: {(config.output_folder)}\n") update_index(config.output_folder, verbose=config.debug, threads=1) + all_recipes = find_all_recipes(args.target, config) # [noqa] console.print("\n[yellow]Assembling all recipes and variants[/yellow]\n") @@ -679,6 +681,7 @@ def run_build(args): except BoaRunBuildException: rerun_build = True except Exception as e: + print(e) raise e else: break diff --git a/boa/core/solver.py b/boa/core/solver.py index 1b259330..c3164bc3 100644 --- a/boa/core/solver.py +++ b/boa/core/solver.py @@ -46,6 +46,7 @@ def get_solver(subdir, output_folder="local"): os.makedirs(pkg_cache, exist_ok=True) if not solver_cache.get(subdir): + print("GET SOLVER") solver_cache[subdir] = MambaSolver([], subdir, output_folder) return solver_cache[subdir], pkg_cache diff --git a/tests/recipes-v2/emscripten_emscripten32/recipe.yaml b/tests/recipes-v2/emscripten_emscripten32/recipe.yaml new file mode 100644 index 00000000..d7c06e8a --- /dev/null +++ b/tests/recipes-v2/emscripten_emscripten32/recipe.yaml @@ -0,0 +1,31 @@ +context: + name: emscripten_emscripten-32 + version: 0.23.10 + +package: + name: '{{ name|lower }}' + version: '{{ version }}' + +build: + number: 0 + +requirements: + + + run: + - emscripten + + +about: + home: emscripten + license: BSD-3-Clause + license_family: BSD + license_file: LICENSE + summary: emscripten + description: emscripten + doc_url: emscripten + dev_url: emscripten + +extra: + recipe-maintainers: + - DerThorsten diff --git a/tests/recipes-v2/xtensor/conda_build_config.yaml b/tests/recipes-v2/xtensor/conda_build_config.yaml new file mode 100644 index 00000000..30f03833 --- /dev/null +++ b/tests/recipes-v2/xtensor/conda_build_config.yaml @@ -0,0 +1,774 @@ +# This differs from target_platform in that it determines what subdir the compiler +# will target, not what subdir the compiler package will be itself. +# For example, we need a win-64 vs2008_win-32 package, so that we compile win-32 +# code on win-64 miniconda. +cross_compiler_target_platform: # [win] + - win-64 # [win] +c_compiler: + - gcc # [linux] + - clang # [osx] + - vs2017 # [win] + - emcc # [emscripten] +c_compiler_version: # [unix] + - 11 # [osx] + - 9 # [linux] + - 7 # [os.environ.get("CF_CUDA_ENABLED", "False") == "True" and linux64] + - 9 # [os.environ.get("CF_CUDA_ENABLED", "False") == "True" and linux64] + - 9 # [os.environ.get("CF_CUDA_ENABLED", "False") == "True" and linux64] + - 9 # [os.environ.get("CF_CUDA_ENABLED", "False") == "True" and linux64] +cxx_compiler: + - gxx # [linux] + - clangxx # [osx] + - vs2017 # [win] + - emcc # [emscripten] +cxx_compiler_version: # [unix] + - 11 # [osx] + - 9 # [linux] + - 7 # [os.environ.get("CF_CUDA_ENABLED", "False") == "True" and linux64] + - 9 # [os.environ.get("CF_CUDA_ENABLED", "False") == "True" and linux64] + - 9 # [os.environ.get("CF_CUDA_ENABLED", "False") == "True" and linux64] + - 9 # [os.environ.get("CF_CUDA_ENABLED", "False") == "True" and linux64] +llvm_openmp: # [osx] + - 11 # [osx] +fortran_compiler: # [unix or win64] + - gfortran # [linux64 or (osx and x86_64)] + - gfortran # [aarch64 or ppc64le or armv7l or s390x] + - flang # [win64] +fortran_compiler_version: # [unix or win64] + - 11 # [osx and arm64] + - 9 # [osx and x86_64] + - 9 # [linux] + - 5 # [win64] + - 7 # [os.environ.get("CF_CUDA_ENABLED", "False") == "True" and linux64] + - 9 # [os.environ.get("CF_CUDA_ENABLED", "False") == "True" and linux64] + - 9 # [os.environ.get("CF_CUDA_ENABLED", "False") == "True" and linux64] + - 9 # [os.environ.get("CF_CUDA_ENABLED", "False") == "True" and linux64] +m2w64_c_compiler: # [win] + - m2w64-toolchain # [win] +m2w64_cxx_compiler: # [win] + - m2w64-toolchain # [win] +m2w64_fortran_compiler: # [win] + - m2w64-toolchain # [win] +CMAKE_GENERATOR: # [win] + - NMake Makefiles # [win] + +cuda_compiler: # [linux64 or win] + - nvcc # [linux64 or win] +cuda_compiler_version: + - None + - 10.2 # [(linux64 or win) and os.environ.get("CF_CUDA_ENABLED", "False") == "True"] + - 11.0 # [(linux64 or win) and os.environ.get("CF_CUDA_ENABLED", "False") == "True"] + - 11.1 # [(linux64 or win) and os.environ.get("CF_CUDA_ENABLED", "False") == "True"] + - 11.2 # [(linux64 or win) and os.environ.get("CF_CUDA_ENABLED", "False") == "True"] +cudnn: + - undefined + - 7 # [(linux64 or win) and os.environ.get("CF_CUDA_ENABLED", "False") == "True"] + - 8 # [(linux64 or win) and os.environ.get("CF_CUDA_ENABLED", "False") == "True"] + - 8 # [(linux64 or win) and os.environ.get("CF_CUDA_ENABLED", "False") == "True"] + - 8 # [(linux64 or win) and os.environ.get("CF_CUDA_ENABLED", "False") == "True"] + +_libgcc_mutex: + - 0.1 conda_forge +# +# Go Compiler Options +# + +# The basic go-compiler with CGO disabled, +# It generates fat binaries without libc dependencies +# The activation scripts will set your CC,CXX and related flags +# to invalid values. +go_compiler: + - go-nocgo +# The go compiler build with CGO enabled. +# It can generate fat binaries that depend on conda's libc. +# You should use this compiler if the underlying +# program needs to link against other C libraries, in which +# case make sure to add 'c,cpp,fortran_compiler' for unix +# and the m2w64 equivalent for windows. +cgo_compiler: + - go-cgo +# The following are helpful variables to simplify go meta.yaml files. +target_goos: + - linux # [linux] + - darwin # [osx] + - windows # [win] +target_goarch: + - amd64 # [x86_64] +target_goexe: + - # [unix] + - .exe # [win] +target_gobin: + - '${PREFIX}/bin/' # [unix] + - '%PREFIX%\bin\' # [win] + +# Rust Compiler Options +rust_compiler: + - rust + +macos_machine: # [osx] + - x86_64-apple-darwin13.4.0 # [osx and x86_64] + - arm64-apple-darwin20.0.0 # [osx and arm64] +MACOSX_DEPLOYMENT_TARGET: # [osx] + - 11.0 # [osx and arm64] + - 10.9 # [osx and x86_64] +target_platform: # [win] + - win-64 # [win] +VERBOSE_AT: + - V=1 +VERBOSE_CM: + - VERBOSE=1 + +# dual build configuration +channel_sources: + - conda-forge # [not s390x] + - https://conda-web.anaconda.org/conda-forge # [s390x] + +channel_targets: + - conda-forge main + +cdt_name: # [linux] + - cos6 # [linux64 and os.environ.get("DEFAULT_LINUX_VERSION", "cos6") == "cos6"] + - cos7 # [linux64 and os.environ.get("DEFAULT_LINUX_VERSION", "cos6") == "cos7"] + - cos7 # [linux and aarch64] + - cos7 # [linux and ppc64le] + - cos7 # [linux and armv7l] + - cos7 # [linux and s390x] + + - cos6 # [linux64 and os.environ.get("CF_CUDA_ENABLED", "False") == "True" and os.environ.get("DEFAULT_LINUX_VERSION", "cos6") == "cos6"] + - cos7 # [linux64 and os.environ.get("CF_CUDA_ENABLED", "False") == "True" and os.environ.get("DEFAULT_LINUX_VERSION", "cos6") == "cos7"] + - cos7 # [linux64 and os.environ.get("CF_CUDA_ENABLED", "False") == "True"] + - cos7 # [linux64 and os.environ.get("CF_CUDA_ENABLED", "False") == "True"] + - cos7 # [linux64 and os.environ.get("CF_CUDA_ENABLED", "False") == "True"] + +docker_image: # [os.environ.get("BUILD_PLATFORM", "").startswith("linux-")] + - quay.io/condaforge/linux-anvil-cos7-x86_64 # [os.environ.get("BUILD_PLATFORM") == "linux-64"] + - quay.io/condaforge/linux-anvil-aarch64 # [os.environ.get("BUILD_PLATFORM") == "linux-aarch64"] + - quay.io/condaforge/linux-anvil-ppc64le # [os.environ.get("BUILD_PLATFORM") == "linux-ppc64le"] + - quay.io/condaforge/linux-anvil-armv7l # [os.environ.get("BUILD_PLATFORM") == "linux-armv7l"] + + - quay.io/condaforge/linux-anvil-cos7-cuda:10.2 # [linux64 and os.environ.get("CF_CUDA_ENABLED", "False") == "True" and os.environ.get("BUILD_PLATFORM") == "linux-64"] + - quay.io/condaforge/linux-anvil-cuda:11.0 # [linux64 and os.environ.get("CF_CUDA_ENABLED", "False") == "True" and os.environ.get("BUILD_PLATFORM") == "linux-64"] + - quay.io/condaforge/linux-anvil-cuda:11.1 # [linux64 and os.environ.get("CF_CUDA_ENABLED", "False") == "True" and os.environ.get("BUILD_PLATFORM") == "linux-64"] + - quay.io/condaforge/linux-anvil-cuda:11.2 # [linux64 and os.environ.get("CF_CUDA_ENABLED", "False") == "True" and os.environ.get("BUILD_PLATFORM") == "linux-64"] + +zip_keys: + - # [unix] + - c_compiler_version # [unix] + - cxx_compiler_version # [unix] + - fortran_compiler_version # [unix] + - cudnn # [linux and os.environ.get("CF_CUDA_ENABLED", "False") == "True"] + - cuda_compiler_version # [linux and os.environ.get("CF_CUDA_ENABLED", "False") == "True"] + - cdt_name # [linux and os.environ.get("CF_CUDA_ENABLED", "False") == "True"] + - docker_image # [linux and os.environ.get("CF_CUDA_ENABLED", "False") == "True" and os.environ.get("BUILD_PLATFORM", "").startswith("linux-")] + - # [win64 and os.environ.get("CF_CUDA_ENABLED", "False") == "True"] + - cudnn # [win64 and os.environ.get("CF_CUDA_ENABLED", "False") == "True"] + - cuda_compiler_version # [win64 and os.environ.get("CF_CUDA_ENABLED", "False") == "True"] + - + - python + - numpy + - python_impl + +# aarch64 specifics because conda-build sets many things to centos 6 +# this can probably be removed when conda-build gets updated defaults +# for aarch64 +cdt_arch: aarch64 # [aarch64] +BUILD: aarch64-conda_cos7-linux-gnu # [aarch64] + +# armv7l specifics because conda-build sets many things to centos 6 +# this can probably be removed when conda-build gets updated defaults +# for aarch64 +cdt_arch: armv7l # [armv7l] +BUILD: armv7-conda_cos7-linux-gnueabihf # [armv7l] + +# TODO: remove these when run_exports are added to the packages. +pin_run_as_build: + arpack: + max_pin: x.x.x + boost: + max_pin: x.x.x + boost-cpp: + max_pin: x.x.x + bzip2: + max_pin: x + cairo: + max_pin: x.x + curl: + max_pin: x + dbus: + max_pin: x + fftw: + max_pin: x + flann: + max_pin: x.x.x + fontconfig: + max_pin: x + freetype: + max_pin: x + gdal: + max_pin: x.x + geotiff: + max_pin: x.x.x + glew: + max_pin: x.x + glpk: + max_pin: x.x + gmp: + max_pin: x + graphviz: + max_pin: x + harfbuzz: + max_pin: x + hdf4: + max_pin: x.x + isl: + max_pin: x.x + jasper: + max_pin: x + jpeg: + max_pin: x + libjpeg_turbo: + max_pin: x + json-c: + max_pin: x.x + jsoncpp: + max_pin: x.x.x + kealib: + max_pin: x.x + krb5: + max_pin: x.x + libblitz: + max_pin: x.x + libcurl: + max_pin: x + libevent: + max_pin: x.x.x + libffi: + max_pin: x.x + libgdal: + max_pin: x.x + libiconv: + max_pin: x.x + libkml: + max_pin: x.x + libpng: + max_pin: x.x + librsvg: + max_pin: x + libsvm: + max_pin: x.x + libtiff: + max_pin: x + libxml2: + max_pin: x.x + libuuid: + max_pin: x + lz4-c: + max_pin: x.x.x + lzo: + max_pin: x + metis: + max_pin: x.x + mpfr: + max_pin: x + netcdf-cxx4: + max_pin: x.x + netcdf-fortran: + max_pin: x.x + nettle: + max_pin: x.x + nlopt: + max_pin: x.x.x + nss: + max_pin: x + nspr: + max_pin: x + occt: + max_pin: x.x + openturns: + max_pin: x.x + openjpeg: + max_pin: x.x + pango: + max_pin: x.x + poppler: + max_pin: x.x + qt: + max_pin: x.x + qtkeychain: + max_pin: x.x + readline: + max_pin: x + r-base: + max_pin: x.x + min_pin: x.x + sox: + max_pin: x.x.x + sqlite: + max_pin: x + tk: + max_pin: x.x + tiledb: + max_pin: x.x + vlfeat: + max_pin: x.x.x + vtk: + max_pin: x.x.x + xz: + max_pin: x.x + zeromq: + max_pin: x.x # [not win] + max_pin: x.x.x # [win] + zlib: + max_pin: x.x + +# Pinning packages + +# blas +libblas: + - 3.8 *netlib # [not (osx and arm64)] + - 3.9 *netlib # [osx and arm64] +libcblas: + - 3.8 *netlib # [not (osx and arm64)] + - 3.9 *netlib # [osx and arm64] +liblapack: + - 3.8 *netlib # [not (osx and arm64)] + - 3.9 *netlib # [osx and arm64] +liblapacke: + - 3.8 *netlib # [not (osx and arm64)] + - 3.9 *netlib # [osx and arm64] +blas_impl: + - openblas + - mkl # [x86 or x86_64] + - blis # [x86 or x86_64] + +abseil_cpp: + - '20210324.2' +alsa_lib: + - 1.2.3 +antic: + - 0.2 +arb: + - '2.21' +arpack: + - 3.7 +arrow_cpp: + - 6.0.1 + - 5.0.0 + - 4.0.1 +assimp: + - 5.2 +aws_c_auth: + - 0.6.10 +aws_c_cal: + - 0.5.12 +aws_c_common: + - 0.6.19 +aws_c_event_stream: + - 0.2.7 +aws_c_http: + - 0.6.10 +aws_c_io: + - 0.10.17 +aws_c_mqtt: + - 0.7.10 +aws_c_s3: + - 0.1.32 +aws_c_sdkutils: + - 0.1.1 +aws_checksums: + - 0.1.12 +aws_crt_cpp: + - 0.17.15 +aws_sdk_cpp: + - 1.9.160 +boost: + - 1.74.0 +boost_cpp: + - 1.74.0 +bzip2: + - 1 +cairo: + - 1.16 +capnproto: + - 0.9.1 +ccr: + - 1.3 +cfitsio: + - 4.0.0 +coin_or_cbc: + - 2.10 +coincbc: + - 2.10 +coin_or_cgl: + - 0.60 +coin_or_clp: + - 1.17 +coin_or_osi: + - 0.108 +coin_or_utils: + - 2.11 +cutensor: + - 1 +curl: + - 7 +davix: + - '0.8' +dbus: + - 1 +exiv2: + - 0.27 +expat: + - 2 +ffmpeg: + - '4.3' +fftw: + - 3 +flann: + - 1.9.1 +fmt: + - '8' +fontconfig: + - 2.13 +freetype: + - 2 +gct: + - 6.2.1629922860 +gf2x: + - '1.3' +gdk_pixbuf: + - 2 +gnuradio_core: + - 3.10.0 +gsl: + - 2.7 +gsoap: + - 2.8.119 +gstreamer: + - 1.18 +gst_plugins_base: + - 1.18 +gdal: + - '3.4' +geos: + - 3.10.2 +geotiff: + - '1.7' +gfal2: + - '2.20' +gflags: + - 2.2 +giflib: + - 5.2 +glew: + - 2.1 +glib: + - '2' +glog: + - '0.5' +glpk: + - 4.65 +gmp: + - 6 +google_cloud_cpp: + - '1.35' +google_cloud_cpp_common: + - 0.25.0 +googleapis_cpp: + - '0.10' +graphviz: + - 2.47 +grpc_cpp: + - '1.43' +harfbuzz: + - '3' +hdf4: + - 4.2 +hdf5: + - 1.10.6 +icu: + - '69' +ipopt: + - 3.14 +isl: + - '0.22' +jasper: + - '2' +jpeg: + - 9 +libjpeg_turbo: + - 2 +json_c: + - 0.15 +jsoncpp: + - 1.9.5 +kealib: + - 1.4 +krb5: + - '1.19' +libarchive: + - 3.5 +libblitz: + - 1.0.2 +libcint: + - '5.1' +libcurl: + - 7 +libcrc32c: + - 1.1 +libdap4: + - 3.20.6 +libeantic: + - 1 +libevent: + - 2.1.10 +libexactreal: + - 2 +libffi: + - '3.3' +libflatsurf: + - 3 +libflint: + - '2.8' +libgdal: + - '3.4' +libgit2: + - '1.3' +libhugetlbfs: + - 2 +libhwy: + - 0.15 +libiconv: + - 1.16 +libintervalxt: + - 3 +libkml: + - 1.3 +libiio: + - 0 +libmatio: + - 1.5.21 +libmicrohttpd: + - 0.9 +libnetcdf: + - 4.8.1 +libopencv: + - 4.5.5 +libpcap: + - '1.10' +libpng: + - 1.6 +libprotobuf: + - '3.18' +librdkafka: + - '1.7' +librsvg: + - 2 +libsecret: + - 0.18 +libspatialindex: + - 1.9.3 +libssh2: + - 1 +libsvm: + - 3.21 +libthrift: + - 0.15.0 +libtiff: + - 4 +libunwind: + - '1.6' +libv8: + - 8.9.83 +libwebp: + - 1 +libwebp_base: + - 1 +libxml2: + - 2.9 +libuuid: + - 2 +libzip: + - 1 +log4cxx: + - 0.11.0 +lz4_c: + - 1.9.3 +lzo: + - 2 +metis: + - 5.1 +mimalloc: + - 1.6.3 +mkl: + - '2021' +mkl_devel: + - '2021' +mpich: + - 3 +mpfr: + - 4 +mumps_mpi: + - 5.2 +mumps_seq: + - 5.2 +nccl: + - 2 +ncurses: + - 6 +netcdf_cxx4: + - 4.3 +netcdf_fortran: + - 4.5 +nettle: + - '3.7' +nodejs: + - '17' + - '16' + - '14' # [not (osx and arm64)] +nss: + - 3 +nspr: + - 4 +nlopt: + - '2.7' +ntl: + - '11.4.3' +# we build for the oldest version possible of numpy for forward compatibility +numpy: + # part of a zip_keys: python, python_impl, numpy + - 1.18 # [not (osx and arm64)] + - 1.18 # [not (osx and arm64)] + - 1.19 # [osx and arm64] + - 1.19 +occt: + - '7.5' +openblas: + - 0.3.* +openexr: + - 2.5 +openjpeg: + - '2.4' +openmpi: + - 4 +openssl: + - 1.1.1 +openturns: + - '1.18' +orc: + - 1.7.2 +pango: + - '1.48' +pari: + - 2.13.* *_pthread +perl: + - 5.32.1 +petsc: + - 3.15 +petsc4py: + - 3.15 +slepc: + - 3.15 +slepc4py: + - 3.15 +pcre: + - '8' +pcre2: + - '10.37' +pixman: + - 0 +poco: + - 1.11.1 +poppler: + - '22.01' +proj: + - 8.2.0 +pybind11_abi: + - 4 +python: + # part of a zip_keys: python, python_impl, numpy + - 3.7.* *_cpython # [not (osx and arm64)] + - 3.8.* *_cpython + - 3.9.* *_cpython +python_impl: + # part of a zip_keys: python, python_impl, numpy + - cpython # [not (osx and arm64)] + - cpython + - cpython +qt: + - 5.12 +qtkeychain: + - '0.12' +re2: + - 2021.11.01 +readline: + - "8" +rocksdb: + - "6.10" +root_base: + - 6.24.6 +ruby: + - 2.5 + - 2.6 +r_base: + - 4.0 + - 4.1 +scotch: + - 6.0.9 +ptscotch: + - 6.0.9 +s2n: + - 1.3.6 +singular: + - 4.2.0.p3 +snappy: + - 1 +soapysdr: + - '0.8' +sox: + - 14.4.2 +spdlog: + - 1.9 +sqlite: + - 3 +suitesparse: + - 5 +superlu_dist: + - 7.1.1 +tbb: + - 2020 +tbb_devel: + - 2020 +thrift_cpp: + - 0.15.0 +tinyxml2: + - 9 +tk: + - 8.6 # [not ppc64le] +tiledb: + - '2.6' +uhd: + - 4.1.0 +vc: # [win] + - 14 # [win] +vlfeat: + - 0.9.20 +volk: + - '2.5' +vtk: + - 9.0.1 +x264: + - '1!161.*' +xerces_c: + - 3.2 +xrootd: + - '5' +xz: + - 5.2 +zeromq: + - 4.3.4 +zlib: + - 1.2 +zlib_ng: + - 2.0 +zstd: + - '1.5' diff --git a/tests/recipes-v2_2/emscripten_emscripten-32/activate.sh b/tests/recipes-v2_2/emscripten_emscripten-32/activate.sh new file mode 100644 index 00000000..f531d71a --- /dev/null +++ b/tests/recipes-v2_2/emscripten_emscripten-32/activate.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +# alias make="emmake make" +# alias Z="emmake make" +$HOME/src/emsdk/emsdk activate latest +source "$HOME/src/emsdk/emsdk_env.sh" \ No newline at end of file diff --git a/tests/recipes-v2_2/emscripten_emscripten-32/build.sh b/tests/recipes-v2_2/emscripten_emscripten-32/build.sh new file mode 100644 index 00000000..122dd92f --- /dev/null +++ b/tests/recipes-v2_2/emscripten_emscripten-32/build.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +# Copy the [de]activate scripts to $PREFIX/etc/conda/[de]activate.d. +# This will allow them to be run on environment activation. +for CHANGE in "activate" "deactivate" +do + mkdir -p "${PREFIX}/etc/conda/${CHANGE}.d" + cp "${RECIPE_DIR}/${CHANGE}.sh" "${PREFIX}/etc/conda/${CHANGE}.d/${PKG_NAME}_${CHANGE}.sh" +done \ No newline at end of file diff --git a/tests/recipes-v2_2/emscripten_emscripten-32/deactivate.sh b/tests/recipes-v2_2/emscripten_emscripten-32/deactivate.sh new file mode 100644 index 00000000..a9bf588e --- /dev/null +++ b/tests/recipes-v2_2/emscripten_emscripten-32/deactivate.sh @@ -0,0 +1 @@ +#!/bin/bash diff --git a/tests/recipes-v2_2/emscripten_emscripten-32/recipe.yaml b/tests/recipes-v2_2/emscripten_emscripten-32/recipe.yaml new file mode 100644 index 00000000..065af621 --- /dev/null +++ b/tests/recipes-v2_2/emscripten_emscripten-32/recipe.yaml @@ -0,0 +1,32 @@ +context: + name: emscripten_emscripten-32 + version: 0.23.10 + +package: + name: '{{ name|lower }}' + version: '{{ version }}' + +build: + number: 0 + +# requirements: + + +# run: +# - emscripten +# - llvm 12.* + + +about: + home: emscripten + license: BSD-3-Clause + license_family: BSD + license_file: LICENSE + summary: emscripten + description: emscripten + doc_url: emscripten + dev_url: emscripten + +extra: + recipe-maintainers: + - DerThorsten diff --git a/tests/recipes-v2_2/xtensor/bld.bat b/tests/recipes-v2_2/xtensor/bld.bat new file mode 100644 index 00000000..3b630bc5 --- /dev/null +++ b/tests/recipes-v2_2/xtensor/bld.bat @@ -0,0 +1,8 @@ +cmake -G "NMake Makefiles" -D BUILD_TESTS=OFF -D CMAKE_INSTALL_PREFIX=%LIBRARY_PREFIX% %SRC_DIR% +if errorlevel 1 exit 1 + +nmake +if errorlevel 1 exit 1 + +nmake install +if errorlevel 1 exit 1 diff --git a/tests/recipes-v2_2/xtensor/build.sh b/tests/recipes-v2_2/xtensor/build.sh new file mode 100644 index 00000000..55b10ecc --- /dev/null +++ b/tests/recipes-v2_2/xtensor/build.sh @@ -0,0 +1,4 @@ +#!/bin/bash + +cmake ${CMAKE_ARGS} -DBUILD_TESTS=OFF -DCMAKE_INSTALL_PREFIX=$PREFIX $SRC_DIR -DCMAKE_INSTALL_LIBDIR=lib +make install diff --git a/tests/recipes-v2_2/xtensor/recipe.yaml b/tests/recipes-v2_2/xtensor/recipe.yaml new file mode 100644 index 00000000..df9ee8f8 --- /dev/null +++ b/tests/recipes-v2_2/xtensor/recipe.yaml @@ -0,0 +1,62 @@ +context: + name: xtensor + version: 0.23.10 + +package: + name: '{{ name|lower }}' + version: '{{ version }}' + +source: + url: https://github.com/xtensor-stack/xtensor/archive/{{ version }}.tar.gz + sha256: 2e770a6d636962eedc868fef4930b919e26efe783cd5d8732c11e14cf72d871c + +build: + number: 0 + +requirements: + build: + - '{{ compiler("cxx") }}' + - cmake + - sel(unix): make + host: + - xtl >=0.7,<0.8 + run: + - xtl >=0.7,<0.8 + run_constrained: + - xsimd >=7.4.8,<8 + +test: + commands: + - sel(unix): test -d ${PREFIX}/include/xtensor + - sel(unix): test -f ${PREFIX}/include/xtensor/xarray.hpp + - sel(unix): test -f ${PREFIX}/lib/cmake/xtensor/xtensorConfig.cmake + - sel(unix): test -f ${PREFIX}/lib/cmake/xtensor/xtensorConfigVersion.cmake + - sel(win): if not exist %LIBRARY_PREFIX%\include\xtensor\xarray.hpp (exit 1) + - sel(win): if not exist %LIBRARY_PREFIX%\lib\cmake\xtensor\xtensorConfig.cmake (exit 1) + - sel(win): if not exist %LIBRARY_PREFIX%\lib\cmake\xtensor\xtensorConfigVersion.cmake (exit 1) + exists: + include: + - xtensor + - xtensor.hpp + - xtensor/xarray.hpp + cmake_find: + - xtensor {{ version }} + pkg_config: + - xtensor + +about: + home: https://github.com/xtensor-stack/xtensor + license: BSD-3-Clause + license_family: BSD + license_file: LICENSE + summary: The C++ tensor algebra library + description: Multi dimensional arrays with broadcasting and lazy computing + doc_url: https://xtensor.readthedocs.io + dev_url: https://github.com/xtensor-stack/xtensor + +extra: + recipe-maintainers: + - SylvainCorlay + - JohanMabille + - wolfv + - davidbrochart From f9224fce8a6949da51bd2fd41ae8b536ecd374b7 Mon Sep 17 00:00:00 2001 From: DerThorsten Date: Thu, 10 Feb 2022 13:06:06 +0100 Subject: [PATCH 02/24] hacky hack --- boa/core/build.py | 84 ++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 83 insertions(+), 1 deletion(-) diff --git a/boa/core/build.py b/boa/core/build.py index 76a4b67b..4c8e2713 100644 --- a/boa/core/build.py +++ b/boa/core/build.py @@ -9,6 +9,7 @@ import fnmatch import io import os +import glob from os.path import isdir, isfile, join import shutil import sys @@ -235,9 +236,70 @@ def select_files(files, include_files, exclude_files): return final_files -def bundle_conda(metadata, initial_files, env, files_selector=None): +def hack_emscripten_generated_js(metadata, filename): + + # the dir where "filename" file lives in (TODO: fixme) + file_base_dir = metadata.config.host_prefix + # os.path.join(os.path.dirname(metadata.config.work_dir),"_build_env") + full_filename = os.path.join(file_base_dir, filename) + + + # we only consider files with a ".js or no suffix" + path = pathlib.Path(filename) + suffix = path.suffix + basename = path.stem + + # files to search + wasm_filename = f'{basename}.wasm' + worker_filename =f'{basename}.worker.js' + + # console.print(f"\n[orange] looking for {wasm_filename} [/orange]\n") + + + added_files = [] + if suffix == ".js" or suffix == "": + + with open(full_filename) as f: + try: + content = f.read() + if wasm_filename in content: + console.print(f"\n[red] checking {filename} is referencing WASM [/red]\n") + # search the whole work dir for that file... + + # pathname = os.path.join(metadata.config.work_dir + f"/**/{wasm_filename}") + + # find wasm file and worker + n_found = 0 + for root, subFolders, files in os.walk(metadata.config.work_dir, followlinks=False): + for file in files: + if file == wasm_filename: + console.print(f"\n[red] FOUND WASM [/red]\n") + shutil.copy2(src=os.path.join(root,file), + dst=os.path.join(metadata.config.host_prefix,'bin',file)) + added_files.append(f'bin/{file}') + n_found += 1 + if file == worker_filename: + console.print(f"\n[red] FOUND WORKER [/red]\n") + shutil.copy2(src=os.path.join(root,file), + dst=os.path.join(metadata.config.host_prefix,'bin',file)) + added_files.append(f'bin/{file}') + n_found += 1 + if n_found == 2: + break + + + + + except UnicodeDecodeError: + return added_files + return added_files + +def bundle_conda(metadata, initial_files, env, files_selector=None): + console.print(f"\n[green]initial_files {initial_files}[/green]\n") files = post_process_files(metadata, initial_files) + console.print(f"\n[green]files {initial_files}[/green]\n") + # first filter is so that info_files does not pick up ignored files files = utils.filter_files(files, prefix=metadata.config.host_prefix) @@ -246,6 +308,20 @@ def bundle_conda(metadata, initial_files, env, files_selector=None): files, files_selector.get("include"), files_selector.get("exclude") ) + extra_files = [] + # todo add emscripten selector + # check for binary files + for f in sorted(files): + + if(f.startswith("bin/")): + added_files = hack_emscripten_generated_js(metadata, f) + for f in added_files: + console.print(f"[red] added missing file {f} [/red]\n") + extra_files += added_files + files = sorted(files + extra_files) + + + console.print(f"\n[yellow]Adding files for {metadata.name()}[/yellow]\n") if files: for f in sorted(files): @@ -278,6 +354,8 @@ def bundle_conda(metadata, initial_files, env, files_selector=None): files = select_files(files, include_files, files_selector.get("exclude")) basename = metadata.dist() + print(f"{ metadata.config.work_dir=}") + print(f"{basename=} { metadata.config.host_prefix=}") tmp_archives = [] final_outputs = [] ext = ".tar.bz2" @@ -631,6 +709,8 @@ def build( utils.rm_rf(m.config.info_dir) files_before_script = utils.prefix_files(prefix=m.config.host_prefix) + print("files_before_script",files_before_script) + with open(join(m.config.build_folder, "prefix_files.txt"), "w") as f: f.write("\n".join(sorted(list(files_before_script)))) f.write("\n") @@ -644,9 +724,11 @@ def build( utils.rm_rf(m.config.host_prefix) return + print(f'\n\n{m.output.sections["files"]=}') final_outputs = bundle_conda( m, files_before_script, env, m.output.sections["files"] ) + print("final_outputs",final_outputs) return final_outputs except subprocess.CalledProcessError: ext = "bat" if utils.on_win else "sh" From 210a62d3768411df9846c355e39f60419ea5de54 Mon Sep 17 00:00:00 2001 From: DerThorsten Date: Thu, 17 Feb 2022 10:26:52 +0100 Subject: [PATCH 03/24] emscripten wip --- boa/core/build.py | 96 +++++++++---------- .../emscripten_emscripten32/recipe.yaml | 31 ------ .../emscripten_emscripten-32/activate.sh | 6 -- .../emscripten_emscripten-32/build.sh | 9 -- .../emscripten_emscripten-32/deactivate.sh | 1 - .../emscripten_emscripten-32/recipe.yaml | 32 ------- tests/recipes-v2_2/xtensor/bld.bat | 8 -- tests/recipes-v2_2/xtensor/build.sh | 4 - tests/recipes-v2_2/xtensor/recipe.yaml | 62 ------------ 9 files changed, 48 insertions(+), 201 deletions(-) delete mode 100644 tests/recipes-v2/emscripten_emscripten32/recipe.yaml delete mode 100644 tests/recipes-v2_2/emscripten_emscripten-32/activate.sh delete mode 100644 tests/recipes-v2_2/emscripten_emscripten-32/build.sh delete mode 100644 tests/recipes-v2_2/emscripten_emscripten-32/deactivate.sh delete mode 100644 tests/recipes-v2_2/emscripten_emscripten-32/recipe.yaml delete mode 100644 tests/recipes-v2_2/xtensor/bld.bat delete mode 100644 tests/recipes-v2_2/xtensor/build.sh delete mode 100644 tests/recipes-v2_2/xtensor/recipe.yaml diff --git a/boa/core/build.py b/boa/core/build.py index a8fbaedb..c4b8863b 100644 --- a/boa/core/build.py +++ b/boa/core/build.py @@ -238,62 +238,60 @@ def select_files(files, include_files, exclude_files): def hack_emscripten_generated_js(metadata, filename): - # the dir where "filename" file lives in (TODO: fixme) file_base_dir = metadata.config.host_prefix - # os.path.join(os.path.dirname(metadata.config.work_dir),"_build_env") full_filename = os.path.join(file_base_dir, filename) - - - # we only consider files with a ".js or no suffix" - path = pathlib.Path(filename) - suffix = path.suffix - basename = path.stem - - # files to search - wasm_filename = f'{basename}.wasm' - worker_filename =f'{basename}.worker.js' - - # console.print(f"\n[orange] looking for {wasm_filename} [/orange]\n") - - added_files = [] - if suffix == ".js" or suffix == "": - - - with open(full_filename) as f: - try: - content = f.read() - if wasm_filename in content: - console.print(f"\n[red] checking {filename} is referencing WASM [/red]\n") - # search the whole work dir for that file... - - # pathname = os.path.join(metadata.config.work_dir + f"/**/{wasm_filename}") - - # find wasm file and worker - n_found = 0 - for root, subFolders, files in os.walk(metadata.config.work_dir, followlinks=False): - for file in files: - if file == wasm_filename: - console.print(f"\n[red] FOUND WASM [/red]\n") - shutil.copy2(src=os.path.join(root,file), + with open(full_filename) as f: + try: + content = f.read() + is_emscripten_generated_wasm = False + + if 'wasmBinaryFile = "' in content: + minified = False + is_emscripten_generated_wasm = True + split_on = 'wasmBinaryFile = "' + elif 'wasmBinaryFile="' in content: + minified = True + is_emscripten_generated_wasm = True + split_on = 'wasmBinaryFile="' + + + if is_emscripten_generated_wasm: + + console.print(f"\n[red] FOUND EMSCRIPTEN GENERATED [/red]\n") + + res = content.split(split_on) + assert len(res) == 2 + res2 = res[1].split('.wasm') + assert len(res2) == 2 + + + wasm_filename = res2[0] + ".wasm" + worker_filename = res[0] + ".worker.js" + found = 0 + for root, subFolders, files in os.walk(metadata.config.work_dir, followlinks=False): + for file in files: + if file == wasm_filename: + console.print(f"\n[red] FOUND WASM [/red]\n") + shutil.copy2(src=os.path.join(root,file), dst=os.path.join(metadata.config.host_prefix,'bin',file)) - added_files.append(f'bin/{file}') - n_found += 1 - if file == worker_filename: - console.print(f"\n[red] FOUND WORKER [/red]\n") - shutil.copy2(src=os.path.join(root,file), + added_files.append(f'bin/{file}') + found += 1 + if file == worker_filename: + console.print(f"\n[red] FOUND WORKER [/red]\n") + shutil.copy2(src=os.path.join(root,file), dst=os.path.join(metadata.config.host_prefix,'bin',file)) - added_files.append(f'bin/{file}') - n_found += 1 - if n_found == 2: - break + added_files.append(f'bin/{file}') + found += 1 + if found == 2: + break + except UnicodeDecodeError: + return added_files + return added_files - except UnicodeDecodeError: - return added_files - return added_files def bundle_conda(metadata, initial_files, env, files_selector=None): console.print(f"\n[green]initial_files {initial_files}[/green]\n") @@ -523,7 +521,9 @@ def write_build_scripts(m, script, build_file): if build_file and isfile(build_file) and build_file.endswith(".py"): pyexe = sys.executable - bf.write(f"{pyexe} $BUILD_PREFIX/bitfurnace/runner.py {build_file}") + + bf.write(f"{pyexe} $HOME/src/bitfurnace/bitfurnace/runner.py {build_file}") + # bf.write(f"{pyexe} $BUILD_PREFIX/bitfurnace/runner.py {build_file}") elif build_file and isfile(build_file): bf.write(open(build_file).read()) elif script: diff --git a/tests/recipes-v2/emscripten_emscripten32/recipe.yaml b/tests/recipes-v2/emscripten_emscripten32/recipe.yaml deleted file mode 100644 index d7c06e8a..00000000 --- a/tests/recipes-v2/emscripten_emscripten32/recipe.yaml +++ /dev/null @@ -1,31 +0,0 @@ -context: - name: emscripten_emscripten-32 - version: 0.23.10 - -package: - name: '{{ name|lower }}' - version: '{{ version }}' - -build: - number: 0 - -requirements: - - - run: - - emscripten - - -about: - home: emscripten - license: BSD-3-Clause - license_family: BSD - license_file: LICENSE - summary: emscripten - description: emscripten - doc_url: emscripten - dev_url: emscripten - -extra: - recipe-maintainers: - - DerThorsten diff --git a/tests/recipes-v2_2/emscripten_emscripten-32/activate.sh b/tests/recipes-v2_2/emscripten_emscripten-32/activate.sh deleted file mode 100644 index f531d71a..00000000 --- a/tests/recipes-v2_2/emscripten_emscripten-32/activate.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash - -# alias make="emmake make" -# alias Z="emmake make" -$HOME/src/emsdk/emsdk activate latest -source "$HOME/src/emsdk/emsdk_env.sh" \ No newline at end of file diff --git a/tests/recipes-v2_2/emscripten_emscripten-32/build.sh b/tests/recipes-v2_2/emscripten_emscripten-32/build.sh deleted file mode 100644 index 122dd92f..00000000 --- a/tests/recipes-v2_2/emscripten_emscripten-32/build.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/bash - -# Copy the [de]activate scripts to $PREFIX/etc/conda/[de]activate.d. -# This will allow them to be run on environment activation. -for CHANGE in "activate" "deactivate" -do - mkdir -p "${PREFIX}/etc/conda/${CHANGE}.d" - cp "${RECIPE_DIR}/${CHANGE}.sh" "${PREFIX}/etc/conda/${CHANGE}.d/${PKG_NAME}_${CHANGE}.sh" -done \ No newline at end of file diff --git a/tests/recipes-v2_2/emscripten_emscripten-32/deactivate.sh b/tests/recipes-v2_2/emscripten_emscripten-32/deactivate.sh deleted file mode 100644 index a9bf588e..00000000 --- a/tests/recipes-v2_2/emscripten_emscripten-32/deactivate.sh +++ /dev/null @@ -1 +0,0 @@ -#!/bin/bash diff --git a/tests/recipes-v2_2/emscripten_emscripten-32/recipe.yaml b/tests/recipes-v2_2/emscripten_emscripten-32/recipe.yaml deleted file mode 100644 index 065af621..00000000 --- a/tests/recipes-v2_2/emscripten_emscripten-32/recipe.yaml +++ /dev/null @@ -1,32 +0,0 @@ -context: - name: emscripten_emscripten-32 - version: 0.23.10 - -package: - name: '{{ name|lower }}' - version: '{{ version }}' - -build: - number: 0 - -# requirements: - - -# run: -# - emscripten -# - llvm 12.* - - -about: - home: emscripten - license: BSD-3-Clause - license_family: BSD - license_file: LICENSE - summary: emscripten - description: emscripten - doc_url: emscripten - dev_url: emscripten - -extra: - recipe-maintainers: - - DerThorsten diff --git a/tests/recipes-v2_2/xtensor/bld.bat b/tests/recipes-v2_2/xtensor/bld.bat deleted file mode 100644 index 3b630bc5..00000000 --- a/tests/recipes-v2_2/xtensor/bld.bat +++ /dev/null @@ -1,8 +0,0 @@ -cmake -G "NMake Makefiles" -D BUILD_TESTS=OFF -D CMAKE_INSTALL_PREFIX=%LIBRARY_PREFIX% %SRC_DIR% -if errorlevel 1 exit 1 - -nmake -if errorlevel 1 exit 1 - -nmake install -if errorlevel 1 exit 1 diff --git a/tests/recipes-v2_2/xtensor/build.sh b/tests/recipes-v2_2/xtensor/build.sh deleted file mode 100644 index 55b10ecc..00000000 --- a/tests/recipes-v2_2/xtensor/build.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/bash - -cmake ${CMAKE_ARGS} -DBUILD_TESTS=OFF -DCMAKE_INSTALL_PREFIX=$PREFIX $SRC_DIR -DCMAKE_INSTALL_LIBDIR=lib -make install diff --git a/tests/recipes-v2_2/xtensor/recipe.yaml b/tests/recipes-v2_2/xtensor/recipe.yaml deleted file mode 100644 index df9ee8f8..00000000 --- a/tests/recipes-v2_2/xtensor/recipe.yaml +++ /dev/null @@ -1,62 +0,0 @@ -context: - name: xtensor - version: 0.23.10 - -package: - name: '{{ name|lower }}' - version: '{{ version }}' - -source: - url: https://github.com/xtensor-stack/xtensor/archive/{{ version }}.tar.gz - sha256: 2e770a6d636962eedc868fef4930b919e26efe783cd5d8732c11e14cf72d871c - -build: - number: 0 - -requirements: - build: - - '{{ compiler("cxx") }}' - - cmake - - sel(unix): make - host: - - xtl >=0.7,<0.8 - run: - - xtl >=0.7,<0.8 - run_constrained: - - xsimd >=7.4.8,<8 - -test: - commands: - - sel(unix): test -d ${PREFIX}/include/xtensor - - sel(unix): test -f ${PREFIX}/include/xtensor/xarray.hpp - - sel(unix): test -f ${PREFIX}/lib/cmake/xtensor/xtensorConfig.cmake - - sel(unix): test -f ${PREFIX}/lib/cmake/xtensor/xtensorConfigVersion.cmake - - sel(win): if not exist %LIBRARY_PREFIX%\include\xtensor\xarray.hpp (exit 1) - - sel(win): if not exist %LIBRARY_PREFIX%\lib\cmake\xtensor\xtensorConfig.cmake (exit 1) - - sel(win): if not exist %LIBRARY_PREFIX%\lib\cmake\xtensor\xtensorConfigVersion.cmake (exit 1) - exists: - include: - - xtensor - - xtensor.hpp - - xtensor/xarray.hpp - cmake_find: - - xtensor {{ version }} - pkg_config: - - xtensor - -about: - home: https://github.com/xtensor-stack/xtensor - license: BSD-3-Clause - license_family: BSD - license_file: LICENSE - summary: The C++ tensor algebra library - description: Multi dimensional arrays with broadcasting and lazy computing - doc_url: https://xtensor.readthedocs.io - dev_url: https://github.com/xtensor-stack/xtensor - -extra: - recipe-maintainers: - - SylvainCorlay - - JohanMabille - - wolfv - - davidbrochart From ab5013e4678eb36d5dea49c061f50f2c352126e9 Mon Sep 17 00:00:00 2001 From: DerThorsten Date: Wed, 2 Mar 2022 09:57:07 +0100 Subject: [PATCH 04/24] wip --- boa/core/build.py | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/boa/core/build.py b/boa/core/build.py index c4b8863b..503ccf10 100644 --- a/boa/core/build.py +++ b/boa/core/build.py @@ -294,10 +294,7 @@ def hack_emscripten_generated_js(metadata, filename): def bundle_conda(metadata, initial_files, env, files_selector=None): - console.print(f"\n[green]initial_files {initial_files}[/green]\n") files = post_process_files(metadata, initial_files) - console.print(f"\n[green]files {initial_files}[/green]\n") - # first filter is so that info_files does not pick up ignored files files = utils.filter_files(files, prefix=metadata.config.host_prefix) @@ -316,7 +313,9 @@ def bundle_conda(metadata, initial_files, env, files_selector=None): for f in added_files: console.print(f"[red] added missing file {f} [/red]\n") extra_files += added_files + files = sorted(files + extra_files) + files = list(dict.fromkeys(files)) @@ -352,8 +351,7 @@ def bundle_conda(metadata, initial_files, env, files_selector=None): files = select_files(files, include_files, files_selector.get("exclude")) basename = metadata.dist() - print(f"{ metadata.config.work_dir=}") - print(f"{basename=} { metadata.config.host_prefix=}") + tmp_archives = [] final_outputs = [] ext = ".tar.bz2" @@ -521,9 +519,9 @@ def write_build_scripts(m, script, build_file): if build_file and isfile(build_file) and build_file.endswith(".py"): pyexe = sys.executable - - bf.write(f"{pyexe} $HOME/src/bitfurnace/bitfurnace/runner.py {build_file}") - # bf.write(f"{pyexe} $BUILD_PREFIX/bitfurnace/runner.py {build_file}") + # sys.path.append("$BUILD_PREFIX/") + # bf.write(f"{pyexe} $HOME/src/bitfurnace/bitfurnace/runner.py {build_file}") + bf.write(f"{pyexe} $BUILD_PREFIX/bitfurnace/runner.py {build_file}") elif build_file and isfile(build_file): bf.write(open(build_file).read()) elif script: @@ -718,8 +716,7 @@ def build( utils.rm_rf(m.config.info_dir) files_before_script = utils.prefix_files(prefix=m.config.host_prefix) - print("files_before_script",files_before_script) - + with open(join(m.config.build_folder, "prefix_files.txt"), "w") as f: f.write("\n".join(sorted(list(files_before_script)))) f.write("\n") @@ -733,11 +730,9 @@ def build( utils.rm_rf(m.config.host_prefix) return - print(f'\n\n{m.output.sections["files"]=}') final_outputs = bundle_conda( m, files_before_script, env, m.output.sections["files"] ) - print("final_outputs",final_outputs) return final_outputs except subprocess.CalledProcessError: ext = "bat" if utils.on_win else "sh" From 3500b3d2623da158819df385e57d4db167f14c81 Mon Sep 17 00:00:00 2001 From: DerThorsten Date: Sun, 13 Mar 2022 15:53:20 +0100 Subject: [PATCH 05/24] hack to not install pip by default --- boa/cli/boa.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/boa/cli/boa.py b/boa/cli/boa.py index 0d85f59a..6e163633 100644 --- a/boa/cli/boa.py +++ b/boa/cli/boa.py @@ -7,6 +7,7 @@ from boa.core.config import init_global_config from mamba.utils import init_api_context +import libmambapy as api banner = r""" _ @@ -103,6 +104,8 @@ def main(config=None): command = args.command init_api_context() + api_ctx = api.Context() + api_ctx.add_pip_as_python_dependency = False init_global_config(args) from boa.core.run_build import run_build From 38e20c4ff8580821dfb2f6292274fd15bc30deaa Mon Sep 17 00:00:00 2001 From: DerThorsten Date: Mon, 14 Mar 2022 09:58:28 +0100 Subject: [PATCH 06/24] hack to set emsdk dir --- boa/core/build.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/boa/core/build.py b/boa/core/build.py index 503ccf10..5f404f32 100644 --- a/boa/core/build.py +++ b/boa/core/build.py @@ -6,7 +6,7 @@ """ from __future__ import absolute_import, division, print_function -import fnmatch +import fnmatchF import io import os import glob @@ -451,6 +451,14 @@ def write_build_scripts(m, script, build_file): env["CONDA_BUILD_STATE"] = "BUILD" + # HACK + emsdk_dir = os.environ.get('EMSDK_DIR') + print(f"\n\n\nEMSDK DIR {emsdk_dir}") + if emsdk_dir is None: + raise RuntimeError("emsdk_dir is none") + env["EMSDK_DIR"] = emsdk_dir + + # forcing shiny colors everywhere env["CLICOLOR_FORCE"] = 1 env["AM_COLOR_TESTS"] = "always" From 252287a8e9e34515ef261f60b9162d7fd96a364c Mon Sep 17 00:00:00 2001 From: DerThorsten Date: Mon, 14 Mar 2022 10:06:46 +0100 Subject: [PATCH 07/24] fixed --- boa/core/build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/boa/core/build.py b/boa/core/build.py index 5f404f32..06c59a40 100644 --- a/boa/core/build.py +++ b/boa/core/build.py @@ -6,7 +6,7 @@ """ from __future__ import absolute_import, division, print_function -import fnmatchF +import fnmatch import io import os import glob From 9ad2be76063a42f2578eff7309ccb003b2b5e8c4 Mon Sep 17 00:00:00 2001 From: DerThorsten Date: Mon, 14 Mar 2022 10:09:14 +0100 Subject: [PATCH 08/24] renaming EMSDK_DIR --- boa/core/build.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/boa/core/build.py b/boa/core/build.py index 06c59a40..d8d9cbb4 100644 --- a/boa/core/build.py +++ b/boa/core/build.py @@ -452,11 +452,11 @@ def write_build_scripts(m, script, build_file): env["CONDA_BUILD_STATE"] = "BUILD" # HACK - emsdk_dir = os.environ.get('EMSDK_DIR') - print(f"\n\n\nEMSDK DIR {emsdk_dir}") + emsdk_dir = os.environ.get('CONDA_EMSDK_DIR') + print(f"\n\n\nCONDA_EMSDK_DIR {emsdk_dir}") if emsdk_dir is None: raise RuntimeError("emsdk_dir is none") - env["EMSDK_DIR"] = emsdk_dir + env["CONDA_EMSDK_DIR"] = emsdk_dir # forcing shiny colors everywhere From d90c12bdd165e977073eb6694a806f6a5bc30386 Mon Sep 17 00:00:00 2001 From: DerThorsten Date: Fri, 18 Mar 2022 09:26:09 +0100 Subject: [PATCH 09/24] using monkeypatch --- boa/core/build.py | 3 + boa/core/config.py | 2 +- boa/core/jinja_support.py | 2 +- boa/core/metadata.py | 2 +- boa/core/monkeypatch.py | 110 ++++++++++++++++++++++++++++++++++++ boa/core/recipe_handling.py | 2 +- boa/core/recipe_output.py | 2 +- boa/core/render.py | 2 +- boa/core/run_build.py | 2 +- boa/core/solver.py | 2 +- boa/core/test.py | 2 +- boa/core/utils.py | 2 +- boa/core/validation.py | 2 +- 13 files changed, 124 insertions(+), 11 deletions(-) create mode 100644 boa/core/monkeypatch.py diff --git a/boa/core/build.py b/boa/core/build.py index d8d9cbb4..4aafcc7b 100644 --- a/boa/core/build.py +++ b/boa/core/build.py @@ -4,6 +4,9 @@ """ Module that does most of the heavy lifting for the ``conda build`` command. """ + +from .monkeypatch import * + from __future__ import absolute_import, division, print_function import fnmatch diff --git a/boa/core/config.py b/boa/core/config.py index 72e26562..85bae6f4 100644 --- a/boa/core/config.py +++ b/boa/core/config.py @@ -1,6 +1,6 @@ # Copyright (C) 2021, QuantStack # SPDX-License-Identifier: BSD-3-Clause - +from .monkeypatch import * from rich.console import Console boa_config = None diff --git a/boa/core/jinja_support.py b/boa/core/jinja_support.py index 461bb203..57f15bb0 100644 --- a/boa/core/jinja_support.py +++ b/boa/core/jinja_support.py @@ -1,6 +1,6 @@ # Copyright (C) 2021, QuantStack # SPDX-License-Identifier: BSD-3-Clause - +from .monkeypatch import * import os from functools import partial from conda_build.jinja_context import cdt diff --git a/boa/core/metadata.py b/boa/core/metadata.py index 2a48fdeb..492b97bc 100644 --- a/boa/core/metadata.py +++ b/boa/core/metadata.py @@ -1,6 +1,6 @@ # Copyright (C) 2021, QuantStack # SPDX-License-Identifier: BSD-3-Clause - +from .monkeypatch import * from typing import Union, Dict, Iterable, Any, Optional try: diff --git a/boa/core/monkeypatch.py b/boa/core/monkeypatch.py new file mode 100644 index 00000000..c2b78ee7 --- /dev/null +++ b/boa/core/monkeypatch.py @@ -0,0 +1,110 @@ +from conda_build import exceptions, utils, variants, environ +from conda_build.conda_interface import non_x86_linux_machines +from conda_build import metadata +from conda_build import utils + +def ns_cfg(config): + print("MONKEYPATCHED") + # Remember to update the docs of any of this changes + plat = config.host_subdir + print(f"\n\n\n{plat=}\n\n\n") + d = dict( + linux=plat.startswith('linux-'), + linux32=bool(plat == 'linux-32'), + linux64=bool(plat == 'linux-64'), + arm=plat.startswith('linux-arm'), + osx=plat.startswith('osx-'), + emscripten=plat.startswith('emscripten-'), + emscripten32=bool(plat == 'emscripten-32'), + emscripten64=bool(plat == 'emscripten-64'), + unix=plat.startswith(('linux-', 'osx-', 'emscripten-')), + win=plat.startswith('win-'), + win32=bool(plat == 'win-32'), + win64=bool(plat == 'win-64'), + x86=plat.endswith(('-32', '-64')), + x86_64=plat.endswith('-64'), + os=os, + environ=os.environ, + nomkl=bool(int(os.environ.get('FEATURE_NOMKL', False))) + ) + + defaults = variants.get_default_variant(config) + py = config.variant.get('python', defaults['python']) + # there are times when python comes in as a tuple + if not hasattr(py, 'split'): + py = py[0] + # go from "3.6 *_cython" -> "36" + # or from "3.6.9" -> "36" + py = int("".join(py.split(' ')[0].split('.')[:2])) + + d["build_platform"] = config.build_subdir + + d.update(dict(py=py, + py3k=bool(30 <= py < 40), + py2k=bool(20 <= py < 30), + py26=bool(py == 26), + py27=bool(py == 27), + py33=bool(py == 33), + py34=bool(py == 34), + py35=bool(py == 35), + py36=bool(py == 36),)) + + np = config.variant.get('numpy') + if not np: + np = defaults['numpy'] + if config.verbose: + utils.get_logger(__name__).warn("No numpy version specified in conda_build_config.yaml. " + "Falling back to default numpy value of {}".format(defaults['numpy'])) + d['np'] = int("".join(np.split('.')[:2])) + + pl = config.variant.get('perl', defaults['perl']) + d['pl'] = pl + + lua = config.variant.get('lua', defaults['lua']) + d['lua'] = lua + d['luajit'] = bool(lua[0] == "2") + + for machine in non_x86_linux_machines: + d[machine] = bool(plat.endswith('-%s' % machine)) + + for feature, value in feature_list: + d[feature] = value + d.update(os.environ) + + # here we try to do some type conversion for more intuitive usage. Otherwise, + # values like 35 are strings by default, making relational operations confusing. + # We also convert "True" and things like that to booleans. + for k, v in config.variant.items(): + if k not in d: + try: + d[k] = int(v) + except (TypeError, ValueError): + if isinstance(v, string_types) and v.lower() in ('false', 'true'): + v = v.lower() == 'true' + d[k] = v + return d + + +metadata.ns_cfg = ns_cfg + + + +DEFAULT_SUBDIRS = { + "linux-64", + "linux-32", + "linux-s390x", + "linux-ppc64", + "linux-ppc64le", + "linux-armv6l", + "linux-armv7l", + "linux-aarch64", + "win-64", + "win-32", + "osx-64", + "osx-arm64", + "zos-z", + "noarch", + "emscripten-32" +} + +utils.DEFAULT_SUBDIRS = DEFAULT_SUBDIRS \ No newline at end of file diff --git a/boa/core/recipe_handling.py b/boa/core/recipe_handling.py index 9a870d80..920a33a7 100644 --- a/boa/core/recipe_handling.py +++ b/boa/core/recipe_handling.py @@ -1,6 +1,6 @@ # Copyright (C) 2021, QuantStack # SPDX-License-Identifier: BSD-3-Clause - +from .monkeypatch import * import os import re import time diff --git a/boa/core/recipe_output.py b/boa/core/recipe_output.py index 728f2c1e..0cc71d8b 100644 --- a/boa/core/recipe_output.py +++ b/boa/core/recipe_output.py @@ -1,6 +1,6 @@ # Copyright (C) 2021, QuantStack # SPDX-License-Identifier: BSD-3-Clause - +from .monkeypatch import * from boa.core.solver import get_solver import copy import re diff --git a/boa/core/render.py b/boa/core/render.py index 2ed853f2..63833d6c 100644 --- a/boa/core/render.py +++ b/boa/core/render.py @@ -1,6 +1,6 @@ # Copyright (C) 2021, QuantStack # SPDX-License-Identifier: BSD-3-Clause - +from .monkeypatch import * from ruamel.yaml import YAML import jinja2 from boa.core.jinja_support import jinja_functions diff --git a/boa/core/run_build.py b/boa/core/run_build.py index 4d517fb9..56cad64c 100644 --- a/boa/core/run_build.py +++ b/boa/core/run_build.py @@ -1,6 +1,6 @@ # Copyright (C) 2021, QuantStack # SPDX-License-Identifier: BSD-3-Clause - +from .monkeypatch import * import os import glob import itertools diff --git a/boa/core/solver.py b/boa/core/solver.py index b63c60ce..98b39418 100644 --- a/boa/core/solver.py +++ b/boa/core/solver.py @@ -1,6 +1,6 @@ # Copyright (C) 2021, QuantStack # SPDX-License-Identifier: BSD-3-Clause - +from .monkeypatch import * import os import tempfile diff --git a/boa/core/test.py b/boa/core/test.py index 23255adf..f94b45ce 100644 --- a/boa/core/test.py +++ b/boa/core/test.py @@ -1,6 +1,6 @@ # Copyright (C) 2021, QuantStack # SPDX-License-Identifier: BSD-3-Clause - +from .monkeypatch import * import json import logging import os diff --git a/boa/core/utils.py b/boa/core/utils.py index e4617409..3d961a71 100644 --- a/boa/core/utils.py +++ b/boa/core/utils.py @@ -1,6 +1,6 @@ # Copyright (C) 2021, QuantStack # SPDX-License-Identifier: BSD-3-Clause - +from .monkeypatch import * import collections import sys import os diff --git a/boa/core/validation.py b/boa/core/validation.py index 823f19e7..727ce26e 100644 --- a/boa/core/validation.py +++ b/boa/core/validation.py @@ -1,6 +1,6 @@ # Copyright (C) 2021, QuantStack # SPDX-License-Identifier: BSD-3-Clause - +from .monkeypatch import * from jsonschema import validate as json_validate import json5 as json from jsonschema.exceptions import ValidationError, SchemaError From bd613a73e62fdc487d7de352b720bede7a8501ea Mon Sep 17 00:00:00 2001 From: DerThorsten Date: Fri, 18 Mar 2022 09:36:15 +0100 Subject: [PATCH 10/24] fixed monkeypatch --- boa/core/build.py | 2 +- boa/core/monkeypatch.py | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/boa/core/build.py b/boa/core/build.py index 4aafcc7b..70564b20 100644 --- a/boa/core/build.py +++ b/boa/core/build.py @@ -5,9 +5,9 @@ Module that does most of the heavy lifting for the ``conda build`` command. """ -from .monkeypatch import * from __future__ import absolute_import, division, print_function +from .monkeypatch import * import fnmatch import io diff --git a/boa/core/monkeypatch.py b/boa/core/monkeypatch.py index c2b78ee7..65f64e0a 100644 --- a/boa/core/monkeypatch.py +++ b/boa/core/monkeypatch.py @@ -2,6 +2,9 @@ from conda_build.conda_interface import non_x86_linux_machines from conda_build import metadata from conda_build import utils +from conda_build.features import feature_list +from conda_build.conda_interface import string_types +import os def ns_cfg(config): print("MONKEYPATCHED") From 8e33dbffbed1dfa49fa273842562720afa4158c6 Mon Sep 17 00:00:00 2001 From: DerThorsten Date: Fri, 18 Mar 2022 10:04:13 +0100 Subject: [PATCH 11/24] improved monkeypatching --- boa/cli/boa.py | 2 +- boa/core/build.py | 29 +++++++++++++++-------------- boa/core/monkeypatch.py | 25 ++++++++++++++++++++++--- 3 files changed, 38 insertions(+), 18 deletions(-) diff --git a/boa/cli/boa.py b/boa/cli/boa.py index 6e163633..e5c01d79 100644 --- a/boa/cli/boa.py +++ b/boa/cli/boa.py @@ -3,7 +3,7 @@ import sys import argparse - +from boa.core.monkeypatch import * from boa.core.config import init_global_config from mamba.utils import init_api_context diff --git a/boa/core/build.py b/boa/core/build.py index 70564b20..1d205204 100644 --- a/boa/core/build.py +++ b/boa/core/build.py @@ -8,7 +8,7 @@ from __future__ import absolute_import, division, print_function from .monkeypatch import * - +import conda_build import fnmatch import io import os @@ -306,19 +306,20 @@ def bundle_conda(metadata, initial_files, env, files_selector=None): files, files_selector.get("include"), files_selector.get("exclude") ) - extra_files = [] - # todo add emscripten selector - # check for binary files - for f in sorted(files): - - if(f.startswith("bin/")): - added_files = hack_emscripten_generated_js(metadata, f) - for f in added_files: - console.print(f"[red] added missing file {f} [/red]\n") - extra_files += added_files + if False: + extra_files = [] + # todo add emscripten selector + # check for binary files + for f in sorted(files): + + if(f.startswith("bin/")): + added_files = hack_emscripten_generated_js(metadata, f) + for f in added_files: + console.print(f"[red] added missing file {f} [/red]\n") + extra_files += added_files - files = sorted(files + extra_files) - files = list(dict.fromkeys(files)) + files = sorted(files + extra_files) + files = list(dict.fromkeys(files)) @@ -704,7 +705,7 @@ def build( if m.skip(): # console.print(utils.get_skip_message(m)) return {} - + print(conda_build.utils.DEFAULT_SUBDIRS) with utils.path_prepended(m.config.build_prefix): env = environ.get_dict(m=m) diff --git a/boa/core/monkeypatch.py b/boa/core/monkeypatch.py index 65f64e0a..25cb7bba 100644 --- a/boa/core/monkeypatch.py +++ b/boa/core/monkeypatch.py @@ -1,11 +1,12 @@ from conda_build import exceptions, utils, variants, environ from conda_build.conda_interface import non_x86_linux_machines from conda_build import metadata -from conda_build import utils +from conda_build import utils,environ from conda_build.features import feature_list from conda_build.conda_interface import string_types -import os +import os +import sys def ns_cfg(config): print("MONKEYPATCHED") # Remember to update the docs of any of this changes @@ -110,4 +111,22 @@ def ns_cfg(config): "emscripten-32" } -utils.DEFAULT_SUBDIRS = DEFAULT_SUBDIRS \ No newline at end of file +utils.DEFAULT_SUBDIRS = DEFAULT_SUBDIRS + + + +def get_shlib_ext(host_platform): + # Return the shared library extension. + if host_platform.startswith('win'): + return '.dll' + elif host_platform in ['osx', 'darwin']: + return '.dylib' + elif host_platform.startswith('linux') or host_platform.startswith("emscripten"): + return '.so' + elif host_platform == 'noarch': + # noarch packages should not contain shared libraries, use the system + # platform if this is requested + return get_shlib_ext(sys.platform) + else: + raise NotImplementedError(host_platform) +environ.get_shlib_ext = get_shlib_ext \ No newline at end of file From abbe51295ea0250ed23cac8c69ac35f2f5bcac34 Mon Sep 17 00:00:00 2001 From: DerThorsten Date: Fri, 18 Mar 2022 10:08:42 +0100 Subject: [PATCH 12/24] conda monkeypatch --- boa/core/monkeypatch.py | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/boa/core/monkeypatch.py b/boa/core/monkeypatch.py index 25cb7bba..33d97be9 100644 --- a/boa/core/monkeypatch.py +++ b/boa/core/monkeypatch.py @@ -1,3 +1,27 @@ +from conda.base import constants + + +KNOWN_SUBDIRS = PLATFORM_DIRECTORIES = ( + "noarch", + "linux-32", + "linux-64", + "linux-aarch64", + "linux-armv6l", + "linux-armv7l", + "linux-ppc64", + "linux-ppc64le", + "linux-s390x", + "osx-64", + "osx-arm64", + "win-32", + "win-64", + "zos-z", + "emscripten-32" +) +constants.KNOWN_SUBDIRS = KNOWN_SUBDIRS +constants.PLATFORM_DIRECTORIES = PLATFORM_DIRECTORIES + + from conda_build import exceptions, utils, variants, environ from conda_build.conda_interface import non_x86_linux_machines from conda_build import metadata @@ -5,6 +29,7 @@ from conda_build.features import feature_list from conda_build.conda_interface import string_types + import os import sys def ns_cfg(config): @@ -129,4 +154,5 @@ def get_shlib_ext(host_platform): return get_shlib_ext(sys.platform) else: raise NotImplementedError(host_platform) -environ.get_shlib_ext = get_shlib_ext \ No newline at end of file +environ.get_shlib_ext = get_shlib_ext + From cd388a37e6bbac3d66796ce8b6bc677fae83d86a Mon Sep 17 00:00:00 2001 From: DerThorsten Date: Fri, 25 Mar 2022 11:46:05 +0100 Subject: [PATCH 13/24] cleanup monkeypatch --- boa/core/monkeypatch.py | 15 ++++++++++----- boa/core/solver.py | 1 + 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/boa/core/monkeypatch.py b/boa/core/monkeypatch.py index 33d97be9..dffe035c 100644 --- a/boa/core/monkeypatch.py +++ b/boa/core/monkeypatch.py @@ -1,6 +1,11 @@ -from conda.base import constants +import os +import sys +############################################### +# CONDA MONKEY-PATCH +############################################### +from conda.base import constants KNOWN_SUBDIRS = PLATFORM_DIRECTORIES = ( "noarch", "linux-32", @@ -22,6 +27,10 @@ constants.PLATFORM_DIRECTORIES = PLATFORM_DIRECTORIES +############################################### +# CONDA-BUILD MONKEY-PATCH +############################################### + from conda_build import exceptions, utils, variants, environ from conda_build.conda_interface import non_x86_linux_machines from conda_build import metadata @@ -30,13 +39,9 @@ from conda_build.conda_interface import string_types -import os -import sys def ns_cfg(config): - print("MONKEYPATCHED") # Remember to update the docs of any of this changes plat = config.host_subdir - print(f"\n\n\n{plat=}\n\n\n") d = dict( linux=plat.startswith('linux-'), linux32=bool(plat == 'linux-32'), diff --git a/boa/core/solver.py b/boa/core/solver.py index 98b39418..0cd892ff 100644 --- a/boa/core/solver.py +++ b/boa/core/solver.py @@ -159,6 +159,7 @@ def replace_channels(self): start_prio = len(self.channels) + len(self.index) for subdir, channel in self.local_index: + print(f"{subdir=} {channel=}") if not subdir.loaded(): continue From 19964be59a18b6e2687e62abe1ec7730b0c16427 Mon Sep 17 00:00:00 2001 From: DerThorsten Date: Tue, 17 May 2022 14:15:10 +0200 Subject: [PATCH 14/24] wip --- boa/core/build.py | 78 +---------------------------------------------- 1 file changed, 1 insertion(+), 77 deletions(-) diff --git a/boa/core/build.py b/boa/core/build.py index 0d3260b1..2168689f 100644 --- a/boa/core/build.py +++ b/boa/core/build.py @@ -239,63 +239,6 @@ def select_files(files, include_files, exclude_files): return final_files -def hack_emscripten_generated_js(metadata, filename): - - file_base_dir = metadata.config.host_prefix - full_filename = os.path.join(file_base_dir, filename) - added_files = [] - with open(full_filename) as f: - try: - content = f.read() - is_emscripten_generated_wasm = False - - if 'wasmBinaryFile = "' in content: - minified = False - is_emscripten_generated_wasm = True - split_on = 'wasmBinaryFile = "' - elif 'wasmBinaryFile="' in content: - minified = True - is_emscripten_generated_wasm = True - split_on = 'wasmBinaryFile="' - - - if is_emscripten_generated_wasm: - - console.print(f"\n[red] FOUND EMSCRIPTEN GENERATED [/red]\n") - - res = content.split(split_on) - assert len(res) == 2 - res2 = res[1].split('.wasm') - assert len(res2) == 2 - - - wasm_filename = res2[0] + ".wasm" - worker_filename = res[0] + ".worker.js" - found = 0 - for root, subFolders, files in os.walk(metadata.config.work_dir, followlinks=False): - for file in files: - if file == wasm_filename: - console.print(f"\n[red] FOUND WASM [/red]\n") - shutil.copy2(src=os.path.join(root,file), - dst=os.path.join(metadata.config.host_prefix,'bin',file)) - added_files.append(f'bin/{file}') - found += 1 - if file == worker_filename: - console.print(f"\n[red] FOUND WORKER [/red]\n") - shutil.copy2(src=os.path.join(root,file), - dst=os.path.join(metadata.config.host_prefix,'bin',file)) - added_files.append(f'bin/{file}') - found += 1 - if found == 2: - break - - except UnicodeDecodeError: - return added_files - - return added_files - - - def bundle_conda(metadata, initial_files, env, files_selector=None): files = post_process_files(metadata, initial_files) @@ -306,23 +249,6 @@ def bundle_conda(metadata, initial_files, env, files_selector=None): files, files_selector.get("include"), files_selector.get("exclude") ) - if False: - extra_files = [] - # todo add emscripten selector - # check for binary files - for f in sorted(files): - - if(f.startswith("bin/")): - added_files = hack_emscripten_generated_js(metadata, f) - for f in added_files: - console.print(f"[red] added missing file {f} [/red]\n") - extra_files += added_files - - files = sorted(files + extra_files) - files = list(dict.fromkeys(files)) - - - console.print(f"\n[yellow]Adding files for {metadata.name()}[/yellow]\n") if files: for f in sorted(files): @@ -456,13 +382,12 @@ def write_build_scripts(m, script, build_file): env["CONDA_BUILD_STATE"] = "BUILD" # HACK - emsdk_dir = os.environ.get('CONDA_EMSDK_DIR') + emsdk_dir = os.environ.get("CONDA_EMSDK_DIR") print(f"\n\n\nCONDA_EMSDK_DIR {emsdk_dir}") if emsdk_dir is None: raise RuntimeError("emsdk_dir is none") env["CONDA_EMSDK_DIR"] = emsdk_dir - # forcing shiny colors everywhere env["CLICOLOR_FORCE"] = 1 env["AM_COLOR_TESTS"] = "always" @@ -728,7 +653,6 @@ def build( utils.rm_rf(m.config.info_dir) files_before_script = utils.prefix_files(prefix=m.config.host_prefix) - with open(join(m.config.build_folder, "prefix_files.txt"), "w") as f: f.write("\n".join(sorted(list(files_before_script)))) f.write("\n") From e1bcfd4596e60ded058667a7bca8abb27ed02af6 Mon Sep 17 00:00:00 2001 From: DerThorsten Date: Tue, 17 May 2022 14:20:01 +0200 Subject: [PATCH 15/24] wip --- boa/core/build.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/boa/core/build.py b/boa/core/build.py index 2168689f..f1b7a534 100644 --- a/boa/core/build.py +++ b/boa/core/build.py @@ -381,9 +381,7 @@ def write_build_scripts(m, script, build_file): env["CONDA_BUILD_STATE"] = "BUILD" - # HACK emsdk_dir = os.environ.get("CONDA_EMSDK_DIR") - print(f"\n\n\nCONDA_EMSDK_DIR {emsdk_dir}") if emsdk_dir is None: raise RuntimeError("emsdk_dir is none") env["CONDA_EMSDK_DIR"] = emsdk_dir @@ -456,8 +454,6 @@ def write_build_scripts(m, script, build_file): if build_file and isfile(build_file) and build_file.endswith(".py"): pyexe = sys.executable - # sys.path.append("$BUILD_PREFIX/") - # bf.write(f"{pyexe} $HOME/src/bitfurnace/bitfurnace/runner.py {build_file}") bf.write(f"{pyexe} $BUILD_PREFIX/bitfurnace/runner.py {build_file}") elif build_file and isfile(build_file): bf.write(open(build_file).read()) @@ -630,7 +626,6 @@ def build( if m.skip(): # console.print(utils.get_skip_message(m)) return {} - print(conda_build.utils.DEFAULT_SUBDIRS) with utils.path_prepended(m.config.build_prefix): env = environ.get_dict(m=m) From b453309fddacde7cca452f975e1ae63226706f54 Mon Sep 17 00:00:00 2001 From: DerThorsten Date: Tue, 17 May 2022 15:59:31 +0200 Subject: [PATCH 16/24] added post build callback --- boa/core/run_build.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/boa/core/run_build.py b/boa/core/run_build.py index b614fc28..cb107137 100644 --- a/boa/core/run_build.py +++ b/boa/core/run_build.py @@ -456,7 +456,6 @@ def run_build(args): console.print(f"Updating build index: {(config.output_folder)}\n") update_index(config.output_folder, verbose=config.debug, threads=1) - all_recipes = find_all_recipes(args.target, config) # [noqa] console.print("\n[yellow]Assembling all recipes and variants[/yellow]\n") @@ -479,6 +478,8 @@ def run_build(args): rerun_build=rerun_build, ) rerun_build = False + if hasattr(args, "post_build_callback"): + args.post_build_callback(recipe) except BoaRunBuildException: rerun_build = True except Exception as e: From d6014153954d21d4d09c397b23d6aa8dbbcf9424 Mon Sep 17 00:00:00 2001 From: DerThorsten Date: Wed, 18 May 2022 09:29:58 +0200 Subject: [PATCH 17/24] improved callback --- boa/core/run_build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/boa/core/run_build.py b/boa/core/run_build.py index cb107137..561b89f9 100644 --- a/boa/core/run_build.py +++ b/boa/core/run_build.py @@ -478,7 +478,7 @@ def run_build(args): rerun_build=rerun_build, ) rerun_build = False - if hasattr(args, "post_build_callback"): + if getattr(args, "post_build_callback", None) is not None: args.post_build_callback(recipe) except BoaRunBuildException: rerun_build = True From 0479d2b190e6596a414a9e8c04a5c9e2313f4e7e Mon Sep 17 00:00:00 2001 From: DerThorsten Date: Wed, 18 May 2022 12:21:45 +0200 Subject: [PATCH 18/24] wip --- boa/core/run_build.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/boa/core/run_build.py b/boa/core/run_build.py index 561b89f9..6189b205 100644 --- a/boa/core/run_build.py +++ b/boa/core/run_build.py @@ -464,7 +464,7 @@ def run_build(args): for recipe in all_recipes: while True: try: - build_recipe( + sorted_outputs = build_recipe( args.command, recipe["recipe_file"], cbc, @@ -479,7 +479,11 @@ def run_build(args): ) rerun_build = False if getattr(args, "post_build_callback", None) is not None: - args.post_build_callback(recipe) + args.post_build_callback( + recipe=recipe, + target_platform=args.target_platform, + sorted_outputs=sorted_outputs, + ) except BoaRunBuildException: rerun_build = True except Exception as e: From 1f6168205a422d0fae840e91cae6b21f5be63e79 Mon Sep 17 00:00:00 2001 From: DerThorsten Date: Mon, 20 Jun 2022 09:46:43 +0200 Subject: [PATCH 19/24] adding final names --- boa/core/run_build.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/boa/core/run_build.py b/boa/core/run_build.py index 6189b205..d48a824e 100644 --- a/boa/core/run_build.py +++ b/boa/core/run_build.py @@ -222,6 +222,9 @@ def build_recipe( # the final metadata sorted_outputs = to_build_tree(ydoc, variants, config, cbc, selected_features) + # the actual filenames like `pyjs-0.1.0-hc96583f_0` without ending (ie the filename without .tar.bz2) + final_names = [] + # then we need to solve and build from the bottom up # we can't first solve all packages without finalizing everything # @@ -309,7 +312,7 @@ def build_recipe( continue final_name = meta.dist() - + final_names.append(final_name) # TODO this doesn't work for noarch! if skip_existing: final_name = meta.dist() @@ -419,7 +422,7 @@ def build_recipe( print("\n\n") console.print(o) - return sorted_outputs + return sorted_outputs,final_names def extract_features(feature_string): @@ -464,7 +467,7 @@ def run_build(args): for recipe in all_recipes: while True: try: - sorted_outputs = build_recipe( + sorted_outputs,final_names = build_recipe( args.command, recipe["recipe_file"], cbc, @@ -483,6 +486,7 @@ def run_build(args): recipe=recipe, target_platform=args.target_platform, sorted_outputs=sorted_outputs, + final_names=final_names ) except BoaRunBuildException: rerun_build = True From 75aeabea8497de02444126ab8f04370a293e919f Mon Sep 17 00:00:00 2001 From: martinRenou Date: Mon, 8 Aug 2022 17:07:32 +0200 Subject: [PATCH 20/24] Add emscripten wasm filetype patch --- boa/core/monkeypatch.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/boa/core/monkeypatch.py b/boa/core/monkeypatch.py index dffe035c..05d8ead2 100644 --- a/boa/core/monkeypatch.py +++ b/boa/core/monkeypatch.py @@ -1,4 +1,4 @@ -import os +import os import sys @@ -37,6 +37,10 @@ from conda_build import utils,environ from conda_build.features import feature_list from conda_build.conda_interface import string_types +from conda_build.post import filetypes_for_platform + + +filetypes_for_platform["emscripten"] = ["wasm"] def ns_cfg(config): @@ -160,4 +164,3 @@ def get_shlib_ext(host_platform): else: raise NotImplementedError(host_platform) environ.get_shlib_ext = get_shlib_ext - From 172c1dcf1957270cff7e756693a0a9bc4ceace4b Mon Sep 17 00:00:00 2001 From: DerThorsten Date: Mon, 17 Oct 2022 09:39:41 +0200 Subject: [PATCH 21/24] make it non opt --- boa/core/build.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/boa/core/build.py b/boa/core/build.py index f1b7a534..5300079c 100644 --- a/boa/core/build.py +++ b/boa/core/build.py @@ -382,9 +382,8 @@ def write_build_scripts(m, script, build_file): env["CONDA_BUILD_STATE"] = "BUILD" emsdk_dir = os.environ.get("CONDA_EMSDK_DIR") - if emsdk_dir is None: - raise RuntimeError("emsdk_dir is none") - env["CONDA_EMSDK_DIR"] = emsdk_dir + if emsdk_dir is not None: + env["CONDA_EMSDK_DIR"] = emsdk_dir # forcing shiny colors everywhere env["CLICOLOR_FORCE"] = 1 From a42b870fe09668cb450e713366736d176d35a275 Mon Sep 17 00:00:00 2001 From: DerThorsten Date: Thu, 1 Dec 2022 10:40:44 +0100 Subject: [PATCH 22/24] updated for new conda-bld --- boa/core/monkeypatch.py | 119 ++++++++++++++++++++++------------------ 1 file changed, 65 insertions(+), 54 deletions(-) diff --git a/boa/core/monkeypatch.py b/boa/core/monkeypatch.py index 05d8ead2..f27a23fb 100644 --- a/boa/core/monkeypatch.py +++ b/boa/core/monkeypatch.py @@ -6,6 +6,7 @@ # CONDA MONKEY-PATCH ############################################### from conda.base import constants + KNOWN_SUBDIRS = PLATFORM_DIRECTORIES = ( "noarch", "linux-32", @@ -21,7 +22,7 @@ "win-32", "win-64", "zos-z", - "emscripten-32" + "emscripten-32", ) constants.KNOWN_SUBDIRS = KNOWN_SUBDIRS constants.PLATFORM_DIRECTORIES = PLATFORM_DIRECTORIES @@ -34,9 +35,13 @@ from conda_build import exceptions, utils, variants, environ from conda_build.conda_interface import non_x86_linux_machines from conda_build import metadata -from conda_build import utils,environ +from conda_build import utils, environ from conda_build.features import feature_list -from conda_build.conda_interface import string_types + +try: + from conda_build.conda_interface import string_types +except ImportError: + string_types = str from conda_build.post import filetypes_for_platform @@ -47,63 +52,69 @@ def ns_cfg(config): # Remember to update the docs of any of this changes plat = config.host_subdir d = dict( - linux=plat.startswith('linux-'), - linux32=bool(plat == 'linux-32'), - linux64=bool(plat == 'linux-64'), - arm=plat.startswith('linux-arm'), - osx=plat.startswith('osx-'), - emscripten=plat.startswith('emscripten-'), - emscripten32=bool(plat == 'emscripten-32'), - emscripten64=bool(plat == 'emscripten-64'), - unix=plat.startswith(('linux-', 'osx-', 'emscripten-')), - win=plat.startswith('win-'), - win32=bool(plat == 'win-32'), - win64=bool(plat == 'win-64'), - x86=plat.endswith(('-32', '-64')), - x86_64=plat.endswith('-64'), + linux=plat.startswith("linux-"), + linux32=bool(plat == "linux-32"), + linux64=bool(plat == "linux-64"), + arm=plat.startswith("linux-arm"), + osx=plat.startswith("osx-"), + emscripten=plat.startswith("emscripten-"), + emscripten32=bool(plat == "emscripten-32"), + emscripten64=bool(plat == "emscripten-64"), + unix=plat.startswith(("linux-", "osx-", "emscripten-")), + win=plat.startswith("win-"), + win32=bool(plat == "win-32"), + win64=bool(plat == "win-64"), + x86=plat.endswith(("-32", "-64")), + x86_64=plat.endswith("-64"), os=os, environ=os.environ, - nomkl=bool(int(os.environ.get('FEATURE_NOMKL', False))) + nomkl=bool(int(os.environ.get("FEATURE_NOMKL", False))), ) defaults = variants.get_default_variant(config) - py = config.variant.get('python', defaults['python']) + py = config.variant.get("python", defaults["python"]) # there are times when python comes in as a tuple - if not hasattr(py, 'split'): + if not hasattr(py, "split"): py = py[0] # go from "3.6 *_cython" -> "36" # or from "3.6.9" -> "36" - py = int("".join(py.split(' ')[0].split('.')[:2])) + py = int("".join(py.split(" ")[0].split(".")[:2])) d["build_platform"] = config.build_subdir - d.update(dict(py=py, - py3k=bool(30 <= py < 40), - py2k=bool(20 <= py < 30), - py26=bool(py == 26), - py27=bool(py == 27), - py33=bool(py == 33), - py34=bool(py == 34), - py35=bool(py == 35), - py36=bool(py == 36),)) - - np = config.variant.get('numpy') + d.update( + dict( + py=py, + py3k=bool(30 <= py < 40), + py2k=bool(20 <= py < 30), + py26=bool(py == 26), + py27=bool(py == 27), + py33=bool(py == 33), + py34=bool(py == 34), + py35=bool(py == 35), + py36=bool(py == 36), + ) + ) + + np = config.variant.get("numpy") if not np: - np = defaults['numpy'] + np = defaults["numpy"] if config.verbose: - utils.get_logger(__name__).warn("No numpy version specified in conda_build_config.yaml. " - "Falling back to default numpy value of {}".format(defaults['numpy'])) - d['np'] = int("".join(np.split('.')[:2])) + utils.get_logger(__name__).warn( + "No numpy version specified in conda_build_config.yaml. " + "Falling back to default numpy value of {}".format(defaults["numpy"]) + ) + d["np"] = int("".join(np.split(".")[:2])) - pl = config.variant.get('perl', defaults['perl']) - d['pl'] = pl + pl = config.variant.get("perl", defaults["perl"]) + d["pl"] = pl - lua = config.variant.get('lua', defaults['lua']) - d['lua'] = lua - d['luajit'] = bool(lua[0] == "2") + lua = config.variant.get("lua", defaults["lua"]) + d["lua"] = lua + d["luajit"] = bool(lua[0] == "2") for machine in non_x86_linux_machines: - d[machine] = bool(plat.endswith('-%s' % machine)) + d[machine] = bool(plat.endswith("-%s" % machine)) for feature, value in feature_list: d[feature] = value @@ -117,8 +128,8 @@ def ns_cfg(config): try: d[k] = int(v) except (TypeError, ValueError): - if isinstance(v, string_types) and v.lower() in ('false', 'true'): - v = v.lower() == 'true' + if isinstance(v, string_types) and v.lower() in ("false", "true"): + v = v.lower() == "true" d[k] = v return d @@ -126,7 +137,6 @@ def ns_cfg(config): metadata.ns_cfg = ns_cfg - DEFAULT_SUBDIRS = { "linux-64", "linux-32", @@ -142,25 +152,26 @@ def ns_cfg(config): "osx-arm64", "zos-z", "noarch", - "emscripten-32" + "emscripten-32", } utils.DEFAULT_SUBDIRS = DEFAULT_SUBDIRS - def get_shlib_ext(host_platform): # Return the shared library extension. - if host_platform.startswith('win'): - return '.dll' - elif host_platform in ['osx', 'darwin']: - return '.dylib' - elif host_platform.startswith('linux') or host_platform.startswith("emscripten"): - return '.so' - elif host_platform == 'noarch': + if host_platform.startswith("win"): + return ".dll" + elif host_platform in ["osx", "darwin"]: + return ".dylib" + elif host_platform.startswith("linux") or host_platform.startswith("emscripten"): + return ".so" + elif host_platform == "noarch": # noarch packages should not contain shared libraries, use the system # platform if this is requested return get_shlib_ext(sys.platform) else: raise NotImplementedError(host_platform) + + environ.get_shlib_ext = get_shlib_ext From 65d2e346f76600b89a7be934a9a381e6d6596c3c Mon Sep 17 00:00:00 2001 From: DerThorsten Date: Thu, 29 Dec 2022 10:25:47 +0100 Subject: [PATCH 23/24] added python api --- boa/pyapi.py | 96 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 96 insertions(+) create mode 100644 boa/pyapi.py diff --git a/boa/pyapi.py b/boa/pyapi.py new file mode 100644 index 00000000..05d0592b --- /dev/null +++ b/boa/pyapi.py @@ -0,0 +1,96 @@ +from typing import Optional, List, Callable + + +import sys +from .core.monkeypatch import * +from .core.config import init_global_config +from ._version import __version__ +from mamba.utils import init_api_context +import libmambapy as api + +from conda_build.conda_interface import cc_conda_build + + +from types import SimpleNamespace + +banner = r""" + _ + | |__ ___ __ _ + | '_ \ / _ \ / _` | + | |_) | (_) | (_| | + |_.__/ \___/ \__,_| +""" + + +def py_build( + recipe_dir: str, + target: Optional[str] = None, + features: Optional[str] = None, + offline: bool = False, + target_platform: Optional[str] = None, + json: bool = False, + debug: bool = False, + variant_config_files=None, + interactive: bool = False, + output_folder: Optional[str] = None, + skip_existing: Optional[str] = None, + no_test: bool = False, + continue_on_failure: bool = False, + conda_build_build_id_pat: bool = False, + conda_build_remove_work_dir: bool = True, + conda_build_keep_old_work: bool = False, + conda_build_prefix_length: int = 255, + croot: bool = False, + pkg_format: int = 1, + zstd_compression_level: int = 22, + post_build_callback: Optional[Callable] = None, + add_pip_as_python_dependency: bool = False, +): + + if target is None: + target = "" + + if variant_config_files is None: + variant_config_files = [] + + if output_folder is None: + output_folder = cc_conda_build.get("output_folder") + + if skip_existing is None: + skip_existing = "default" + + args = SimpleNamespace( + recipe_dir=recipe_dir, + target=target, + features=features, + offline=offline, + target_platform=target_platform, + json=json, + debug=debug, + variant_config_files=variant_config_files, + interactive=interactive, + output_folder=output_folder, + skip_existing=skip_existing, + no_test=no_test, + continue_on_failure=continue_on_failure, + conda_build_build_id_pat=conda_build_build_id_pat, + conda_build_remove_work_dir=conda_build_remove_work_dir, + conda_build_keep_old_work=conda_build_keep_old_work, + conda_build_prefix_length=conda_build_prefix_length, + croot=croot, + pkg_format=pkg_format, + zstd_compression_level=zstd_compression_level, + post_build_callback=post_build_callback, + command="build", + ) + + print("args", args) + + init_api_context() + api_ctx = api.Context() + api_ctx.add_pip_as_python_dependency = add_pip_as_python_dependency + init_global_config(args) + + from boa.core.run_build import run_build + + run_build(args) From c5d52c1940bf7df8c19e6f053bf881b061dae16e Mon Sep 17 00:00:00 2001 From: DerThorsten Date: Fri, 14 Jul 2023 10:08:39 +0200 Subject: [PATCH 24/24] hotfix to remove breakpoint --- boa/core/metadata.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/boa/core/metadata.py b/boa/core/metadata.py index b363bb0d..af67bbe2 100644 --- a/boa/core/metadata.py +++ b/boa/core/metadata.py @@ -220,7 +220,7 @@ def ms_depends(self, typ="run"): # since channel names can be full urls many of these characters are potentially valid for c in "=!@#$%^&*;\"'\\|<>?/": if c in spec.name: - breakpoint() + sys.exit( "Error: bad character '%s' in package name " "dependency '%s'" % (c, spec.name) @@ -239,7 +239,7 @@ def ms_depends(self, typ="run"): parts[1], parts[2], ) - sys.exit(msg) + raise RuntimeError(msg) specs.append(spec) return specs