From ed17b96edab087fc0be88b180e1577b9d6fbf42a Mon Sep 17 00:00:00 2001 From: Arpit Aggarwal Date: Tue, 16 Oct 2018 20:36:19 +0200 Subject: [PATCH 01/60] moved old travis file to ci --- ci/.travis.yml.old | 62 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100644 ci/.travis.yml.old diff --git a/ci/.travis.yml.old b/ci/.travis.yml.old new file mode 100644 index 00000000000..23507297413 --- /dev/null +++ b/ci/.travis.yml.old @@ -0,0 +1,62 @@ +notifications: + email: false + +language: cpp + +cache: + directories: + - $HOME/.ccache + +os: + - linux + +addons: + apt: + sources: + - ubuntu-toolchain-r-test + - llvm-toolchain-precise-3.8 + packages: + - gdb + - g++-4.9 + - gfortran-4.9 + - liblapack-dev + - clang-3.8 + - sox + +branches: + only: + - master + +before_install: + - cat /proc/sys/kernel/core_pattern + - export XROOT=~/xroot + - tools/extras/travis_install_bindeps.sh $XROOT + - export PATH=$XROOT/usr/bin:$PATH + +before_script: + - which ccache + - ccache --version + - ccache --show-stats + - ccache --zero-stats --max-size=3G + +env: + - CI_TARGETS="all ext" # Job1: Build everything. + - CI_TARGETS="test" # Job2: Test libraries. #### ext_test? adds 5min compile, runs 1 test. + +script: + # See http://petereisentraut.blogspot.com/2011/05/ccache-and-clang.html and + # http://peter.eisentraut.org/blog/2014/12/01/ccache-and-clang-part-3/ + # for the explanation why extra switches needed for clang with ccache. + - CXX="ccache clang++-3.8 -Qunused-arguments -fcolor-diagnostics -Wno-tautological-compare" + CFLAGS="" + LDFLAGS="-llapack" + INCDIRS="$XROOT/usr/include" + LIBDIRS="$XROOT/usr/lib" + tools/extras/travis_script.sh +# To troubleshoot cache, add to above: CI_TARGETS=util CCACHE_LOGFILE=~/ccache.log + +before_cache: + - ccache --show-stats + +after_failure: + - tools/extras/travis_show_failures.sh From 84f733059f606ee974c9761538cc64b4240afcaf Mon Sep 17 00:00:00 2001 From: Arpit Aggarwal Date: Tue, 16 Oct 2018 20:36:51 +0200 Subject: [PATCH 02/60] moved old travis file to ci --- .travis.yml | 62 ----------------------------------------------------- 1 file changed, 62 deletions(-) delete mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 23507297413..00000000000 --- a/.travis.yml +++ /dev/null @@ -1,62 +0,0 @@ -notifications: - email: false - -language: cpp - -cache: - directories: - - $HOME/.ccache - -os: - - linux - -addons: - apt: - sources: - - ubuntu-toolchain-r-test - - llvm-toolchain-precise-3.8 - packages: - - gdb - - g++-4.9 - - gfortran-4.9 - - liblapack-dev - - clang-3.8 - - sox - -branches: - only: - - master - -before_install: - - cat /proc/sys/kernel/core_pattern - - export XROOT=~/xroot - - tools/extras/travis_install_bindeps.sh $XROOT - - export PATH=$XROOT/usr/bin:$PATH - -before_script: - - which ccache - - ccache --version - - ccache --show-stats - - ccache --zero-stats --max-size=3G - -env: - - CI_TARGETS="all ext" # Job1: Build everything. - - CI_TARGETS="test" # Job2: Test libraries. #### ext_test? adds 5min compile, runs 1 test. - -script: - # See http://petereisentraut.blogspot.com/2011/05/ccache-and-clang.html and - # http://peter.eisentraut.org/blog/2014/12/01/ccache-and-clang-part-3/ - # for the explanation why extra switches needed for clang with ccache. - - CXX="ccache clang++-3.8 -Qunused-arguments -fcolor-diagnostics -Wno-tautological-compare" - CFLAGS="" - LDFLAGS="-llapack" - INCDIRS="$XROOT/usr/include" - LIBDIRS="$XROOT/usr/lib" - tools/extras/travis_script.sh -# To troubleshoot cache, add to above: CI_TARGETS=util CCACHE_LOGFILE=~/ccache.log - -before_cache: - - ccache --show-stats - -after_failure: - - tools/extras/travis_show_failures.sh From 1804a005e941463d3f0659ac4c179a5a7ea1cb11 Mon Sep 17 00:00:00 2001 From: Arpit Aggarwal Date: Tue, 16 Oct 2018 20:37:35 +0200 Subject: [PATCH 03/60] added Dockerfile and ignore --- .dockerignore | 5 +++++ Dockerfile | 44 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 49 insertions(+) create mode 100644 .dockerignore create mode 100644 Dockerfile diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000000..6bc11796359 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,5 @@ +# dockerignore file to minimalize the size of docker image +# +# Ignore .git and dev directories +.git +dev diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000000..f6af8ecc9aa --- /dev/null +++ b/Dockerfile @@ -0,0 +1,44 @@ +# ---------------------------------------------------------- +# Dockerfile to build base working Ubuntu image +# ---------------------------------------------------------- + +# Set the base image to Ubuntu 16.04 +FROM ubuntu:16.04 + +# File Author / Maintainer +MAINTAINER Arpit Aggarwal + +# Make image to be noninteractive to use with CI +# Comment this line when not using this image with CI +ENV DEBIAN_FRONTEND=noninteractive + +# Set environment variable USER +ENV USER=hero + +# Update the image and install basic packages +RUN apt-get update -qq > /dev/null && apt-get install --assume-yes --no-install-recommends apt-utils ca-certificates curl dbus dialog git sudo wget -qq > /dev/null + +# Add user USER with SUDO previleges and disable SUDO password +RUN addgroup -gid 2000 $USER && \ + adduser --disabled-password --gecos "" -uid 2000 -gid 2000 $USER && \ + addgroup $USER adm && \ + addgroup $USER sudo && \ + echo "%sudo ALL=(ALL) NOPASSWD:ALL" > /etc/sudoers.d/"$USER" + +USER "$USER" + +# Set WORKDIR to home of arpit +WORKDIR /home/"$USER" + +# Copy contents of repository into directory _git in the home of the user +COPY / ./_git/ + +# Change owner of directory to USER from root +RUN sudo chown -R $USER:$USER _git + +# Run setup and install all dependencies +RUN sudo ./_git/install.bash --complete + +# Start container of the Docker image at bash prompt +CMD ["/bin/bash"] + From d9d3e840667e4a585250bf58f8060d3e08159d69 Mon Sep 17 00:00:00 2001 From: Arpit Aggarwal Date: Sat, 3 Nov 2018 10:37:27 +0100 Subject: [PATCH 04/60] removed environment variable checks, new build function --- install.bash | 218 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 218 insertions(+) create mode 100755 install.bash diff --git a/install.bash b/install.bash new file mode 100755 index 00000000000..1e575f50ff8 --- /dev/null +++ b/install.bash @@ -0,0 +1,218 @@ +#!/usr/bin/env bash +# +# Install script for Linux based Speech Recognition System +# +# Arpit Aggarwal +# 2018 TU Eindhoven + +# Usage Document +usage() +{ + echo "-----------------------------------------------------------------------------" + echo -e "\e[35m\e[1m LASeR: Linux Automatic Speech Recognition \e[0m" + echo "-----------------------------------------------------------------------------" + echo "Usage: sudo ./setup [options] " + echo -e "Options:\n \ + -h | --help\n \ + --build\n \ + --install\n \ + --update\n \ + --clean" + echo + echo "-----------------------------------------------------------------------------" +} + +KALDI_REPO="https://github.com/kaldi-asr/kaldi.git" +KALDI="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +ASR_LOG=$KALDI/log + +# Logging function +log() +{ + echo "$(date "+%Y-%m-%d %H:%M:%S"): $1" >> $log_file +} + +# Create Log directory +if [ ! -d "$ASR_LOG" ] +then + mkdir $ASR_LOG +fi + +# Change system timezone to Europe/Amsterdam [May not be required] +# sudo timedatectl set-timezone Europe/Amsterdam + +# Install the required packages and dependencies +sudo apt-get update -qq > /dev/null 2>&1 +sudo apt-get upgrade --assume-yes -qq > /dev/null 2>&1 +sudo apt-get install --assume-yes build-essential git dphys-swapfile python python-scipy sox zip -qq > /dev/null 2>&1 + +# Install Postgresql only if required to +# postgresql postgresql-contrib python3-psycopg2 + +# Install festival for TTS +# festival + +# Kaldi Build (Common to Installation and Update) +_kaldi_build() +{ + # Change exit to return to source check_dependencies and change back once done + sed -i "s|exit|return|g" $KALDI/tools/extras/check_dependencies.sh + source $KALDI/tools/extras/check_dependencies.sh + sed -i "s|return|exit|g" $KALDI/tools/extras/check_dependencies.sh + + # Install dependencies + echo "Installing dependencies" + sudo apt-get install libatlas3-base $debian_packages -qq > /dev/null 2>&1 + sudo ln -s -f bash /bin/sh + + # Build toolkit + echo "Building toolkit" + # Build the tools directory + cd $KALDI/tools + make -j 4 &> $ASR_LOG/make_tools.log + make_tools_status=$( tail -n 1 $ASR_LOG/make_tools.log ) + + if [ "$make_tools_status" != "All done OK." ] + then + echo -e "\e[34m\e[1m Make kaldi/tools failed \e[0m" + exit 1 + fi + + extras/install_irstlm.sh &> $ASR_LOG/install_irstlm.log + install_irstlm_status=$( grep "Installation of IRSTLM finished successfully" $ASR_LOG/install_irstlm.log ) + + if [ -z "$install_irstlm_status" ] + then + echo -e "\e[34m\e[1m Install kaldi/tools/extras/install_irstlm.sh failed \e[0m" + exit 1 + fi + + # Build the src directory + cd $KALDI/src + ./configure --shared &> $ASR_LOG/configure_src.log + configure_src_status=$( grep "SUCCESS" $ASR_LOG/configure_src.log ) + + if [ -z "$configure_src_status" ] + then + echo -e "\e[34m\e[1m Configure kaldi/src failed \e[0m" + exit 1 + fi + + make depend -j 4 > /dev/null + make -j 4 &> $ASR_LOG/make_src.log + make_src_status=$( grep "Done" $ASR_LOG/make_src.log ) + + if [ -z "$make_src_status" ] + then + echo -e "\e[34m\e[1m Make kaldi/src failed \e[0m" + exit 1 + fi + + # Create a STATUS file to monitor installation + echo -e "\e[36m\e[1m Kaldi installation complete \e[0m" + cd $KALDI + echo "ALL OK" > STATUS +} + +# Kaldi Installation +kaldi_install() +{ + echo "Checking for an existing Kaldi-ASR installation in $KALDI" + + if [ ! -d "$KALDI" ] + then + # Clone repository into $KALDI + echo -e "No existing installation found. Cloning from GitHub repository" + git clone $KALDI_REPO $KALDI + _kaldi_build + else + # Read STATUS file. If not "ALL OK", remove directory $KALDI and re-install Kaldi + kaldi_install_status="$(cat $KALDI/STATUS)" + + if [ "$kaldi_install_status" != "ALL OK" ] + then + sudo rm -rf $KALDI + kaldi_install + fi + fi +} + +# Kaldi Update +kaldi_update() +{ + if [ ! -d "$KALDI" ] + then + # Install Kaldi if directory $KALDI not present + kaldi_install + else + # Read STATUS file. If "ALL OK" then update else remove directory $KALDI + # and re-install Kaldi + kaldi_install_status="$(cat $KALDI/STATUS)" + + if [ "$kaldi_install_status" = "ALL OK" ] + then + # Pull changes from the repository + echo "Updating repository from GitHub" + cd $KALDI + git pull + + # Clean existing make + _kaldi_clean + # Build toolkit + _kaldi_build + + echo -e "\e[36m\e[1m Kaldi-ASR update complete \e[0m" + else + sudo rm -rf $KALDI + kaldi_install + fi + fi +} + +# Clean the repository +_kaldi_clean() +{ + # Clean existing make + echo -e "\e[36m\e[1m Cleaning existing make \e[0m" + cd $KALDI/tools + make distclean + cd $KALDI/src + make distclean + + # Remove anyother file not a part of the repository + cd $KALDI + git clean -fdx +} + +# Read Postional Parameters +if [ -z "$1" ] +then + usage +else + while [ "$1" != "" ] + do + case $1 in + --install-kaldi ) + kaldi_install ;; + + --update-kaldi ) + kaldi_update ;; + + --clean ) + _kaldi_clean ;; + + --build ) + _kaldi_build ;; + + -h | --help ) + usage + exit 1 ;; + + * ) + usage + exit 1 ;; + esac + shift + done +fi + From 8a936b5abb3425b64f6a7725dbce25fcb7608a74 Mon Sep 17 00:00:00 2001 From: Arpit Aggarwal Date: Sat, 3 Nov 2018 10:38:29 +0100 Subject: [PATCH 05/60] export root directory --- setup.bash | 5 +++++ 1 file changed, 5 insertions(+) create mode 100755 setup.bash diff --git a/setup.bash b/setup.bash new file mode 100755 index 00000000000..ded6ca17aac --- /dev/null +++ b/setup.bash @@ -0,0 +1,5 @@ +#! /usr/bin/env bash +# +# Environment variables to use LASeR + +export KALDI_ROOT="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" From 3f6f1ccaf4a52b9dd40cb27abdaa59ee91b38bba Mon Sep 17 00:00:00 2001 From: Arpit Aggarwal Date: Sun, 18 Nov 2018 21:14:23 +0100 Subject: [PATCH 06/60] added install of sequitur for voxforge dataset --- install.bash | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/install.bash b/install.bash index 1e575f50ff8..57282f95334 100755 --- a/install.bash +++ b/install.bash @@ -44,7 +44,7 @@ fi # Install the required packages and dependencies sudo apt-get update -qq > /dev/null 2>&1 sudo apt-get upgrade --assume-yes -qq > /dev/null 2>&1 -sudo apt-get install --assume-yes build-essential git dphys-swapfile python python-scipy sox zip -qq > /dev/null 2>&1 +sudo apt-get install --assume-yes build-essential git dphys-swapfile python python-scipy sox swig zip -qq > /dev/null 2>&1 # Install Postgresql only if required to # postgresql postgresql-contrib python3-psycopg2 @@ -78,6 +78,7 @@ _kaldi_build() exit 1 fi + # Install IRSTLM extras/install_irstlm.sh &> $ASR_LOG/install_irstlm.log install_irstlm_status=$( grep "Installation of IRSTLM finished successfully" $ASR_LOG/install_irstlm.log ) @@ -87,6 +88,9 @@ _kaldi_build() exit 1 fi + # Install Sequitur + extras/install_sequitur.sh + # Build the src directory cd $KALDI/src ./configure --shared &> $ASR_LOG/configure_src.log From a1e0444bd8ad1d3235ed96324c938551e5d417dc Mon Sep 17 00:00:00 2001 From: Arpit Aggarwal Date: Fri, 28 Dec 2018 18:03:14 +0100 Subject: [PATCH 07/60] changed exit to return and make jobs to 8 --- install.bash | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/install.bash b/install.bash index 57282f95334..a078a239f42 100755 --- a/install.bash +++ b/install.bash @@ -69,13 +69,13 @@ _kaldi_build() echo "Building toolkit" # Build the tools directory cd $KALDI/tools - make -j 4 &> $ASR_LOG/make_tools.log + make -j 8 &> $ASR_LOG/make_tools.log make_tools_status=$( tail -n 1 $ASR_LOG/make_tools.log ) if [ "$make_tools_status" != "All done OK." ] then echo -e "\e[34m\e[1m Make kaldi/tools failed \e[0m" - exit 1 + return 1 fi # Install IRSTLM @@ -85,7 +85,7 @@ _kaldi_build() if [ -z "$install_irstlm_status" ] then echo -e "\e[34m\e[1m Install kaldi/tools/extras/install_irstlm.sh failed \e[0m" - exit 1 + return 1 fi # Install Sequitur @@ -99,17 +99,17 @@ _kaldi_build() if [ -z "$configure_src_status" ] then echo -e "\e[34m\e[1m Configure kaldi/src failed \e[0m" - exit 1 + return 1 fi - make depend -j 4 > /dev/null - make -j 4 &> $ASR_LOG/make_src.log + make depend -j 8 > /dev/null + make -j 8 &> $ASR_LOG/make_src.log make_src_status=$( grep "Done" $ASR_LOG/make_src.log ) if [ -z "$make_src_status" ] then echo -e "\e[34m\e[1m Make kaldi/src failed \e[0m" - exit 1 + return 1 fi # Create a STATUS file to monitor installation From d82bca658632b8b49ba975d75a0813c9d91390cd Mon Sep 17 00:00:00 2001 From: Arpit Aggarwal Date: Sun, 27 Jan 2019 16:20:41 +0100 Subject: [PATCH 08/60] moved install dependencies to new function --- install.bash | 27 +++++++++------------------ 1 file changed, 9 insertions(+), 18 deletions(-) diff --git a/install.bash b/install.bash index a078a239f42..8c2640aa346 100755 --- a/install.bash +++ b/install.bash @@ -26,34 +26,19 @@ KALDI_REPO="https://github.com/kaldi-asr/kaldi.git" KALDI="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" ASR_LOG=$KALDI/log -# Logging function -log() -{ - echo "$(date "+%Y-%m-%d %H:%M:%S"): $1" >> $log_file -} - # Create Log directory if [ ! -d "$ASR_LOG" ] then mkdir $ASR_LOG fi -# Change system timezone to Europe/Amsterdam [May not be required] -# sudo timedatectl set-timezone Europe/Amsterdam - # Install the required packages and dependencies sudo apt-get update -qq > /dev/null 2>&1 sudo apt-get upgrade --assume-yes -qq > /dev/null 2>&1 -sudo apt-get install --assume-yes build-essential git dphys-swapfile python python-scipy sox swig zip -qq > /dev/null 2>&1 +sudo apt-get install --assume-yes build-essential git python python-scipy sox swig zip -qq > /dev/null 2>&1 -# Install Postgresql only if required to -# postgresql postgresql-contrib python3-psycopg2 - -# Install festival for TTS -# festival - -# Kaldi Build (Common to Installation and Update) -_kaldi_build() +# Kaldi install dependencies +_kaldi_install_dependencies() { # Change exit to return to source check_dependencies and change back once done sed -i "s|exit|return|g" $KALDI/tools/extras/check_dependencies.sh @@ -64,6 +49,12 @@ _kaldi_build() echo "Installing dependencies" sudo apt-get install libatlas3-base $debian_packages -qq > /dev/null 2>&1 sudo ln -s -f bash /bin/sh +} + +# Kaldi Build (Common to Installation and Update) +_kaldi_build() +{ + _kaldi_install_dependencies # Build toolkit echo "Building toolkit" From 982a677e97893229de0438a8b1c947fcdac71dc7 Mon Sep 17 00:00:00 2001 From: Arpit Aggarwal Date: Tue, 29 Jan 2019 19:07:02 +0100 Subject: [PATCH 09/60] suppress output of check_dependencies --- install.bash | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/install.bash b/install.bash index 8c2640aa346..bb5a5f83777 100755 --- a/install.bash +++ b/install.bash @@ -40,13 +40,13 @@ sudo apt-get install --assume-yes build-essential git python python-scipy sox sw # Kaldi install dependencies _kaldi_install_dependencies() { + echo "Checking and installing dependencies..." # Change exit to return to source check_dependencies and change back once done sed -i "s|exit|return|g" $KALDI/tools/extras/check_dependencies.sh - source $KALDI/tools/extras/check_dependencies.sh + source $KALDI/tools/extras/check_dependencies.sh > /dev/null 2>&1 sed -i "s|return|exit|g" $KALDI/tools/extras/check_dependencies.sh # Install dependencies - echo "Installing dependencies" sudo apt-get install libatlas3-base $debian_packages -qq > /dev/null 2>&1 sudo ln -s -f bash /bin/sh } From 92c0b932215d60f18436363352d8d3b48add67ef Mon Sep 17 00:00:00 2001 From: Arpit Aggarwal Date: Tue, 29 Jan 2019 20:40:08 +0100 Subject: [PATCH 10/60] added g++ version check --- install.bash | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/install.bash b/install.bash index bb5a5f83777..6a7c5c37cde 100755 --- a/install.bash +++ b/install.bash @@ -56,6 +56,14 @@ _kaldi_build() { _kaldi_install_dependencies + # Check g++ version before starting build + gpp_version_num=$(g++ --version | grep ^g++ | sed 's/^.* //g' | sed 's/\./ /g' | xargs printf "%d%02d%02d") + if [ $gpp_version_num -gt 70000 ] + then + echo -e "\e[34m\e[1m Unsupported g++ version. Use g++ < 7.0.* \e[0m" + return 1 + fi + # Build toolkit echo "Building toolkit" # Build the tools directory From 29c59c5c73bcd0d163e685d64f5c53d489995b60 Mon Sep 17 00:00:00 2001 From: Arpit Aggarwal Date: Tue, 29 Jan 2019 22:44:23 +0100 Subject: [PATCH 11/60] added srilm and suppressed output of sequitur --- install.bash | 32 ++++++++++++++++++++++++++++---- 1 file changed, 28 insertions(+), 4 deletions(-) diff --git a/install.bash b/install.bash index 6a7c5c37cde..9f710faf1df 100755 --- a/install.bash +++ b/install.bash @@ -65,7 +65,7 @@ _kaldi_build() fi # Build toolkit - echo "Building toolkit" + echo "Building toolkit..." # Build the tools directory cd $KALDI/tools make -j 8 &> $ASR_LOG/make_tools.log @@ -76,19 +76,41 @@ _kaldi_build() echo -e "\e[34m\e[1m Make kaldi/tools failed \e[0m" return 1 fi + echo " - Built tools" # Install IRSTLM extras/install_irstlm.sh &> $ASR_LOG/install_irstlm.log - install_irstlm_status=$( grep "Installation of IRSTLM finished successfully" $ASR_LOG/install_irstlm.log ) + install_irstlm_status=$(grep "Installation of IRSTLM finished successfully" $ASR_LOG/install_irstlm.log ) if [ -z "$install_irstlm_status" ] then echo -e "\e[34m\e[1m Install kaldi/tools/extras/install_irstlm.sh failed \e[0m" return 1 fi + echo " - Built IRSTLM" # Install Sequitur - extras/install_sequitur.sh + extras/install_sequitur.sh &> $ASR_LOG/install_sequitur.log + install_sequitur_status=$(grep "Installation of SEQUITUR finished successfully" $ASR_LOG/install_sequitur.log) + + if [ -z "$install_sequitur_status" ] + then + echo -e "\e[34m\e[1m Install kaldi/tools/extras/install_sequitur.sh failed \e[0m" + return 1 + fi + echo " - Built SEQUITUR" + + # Install SRILM + wget https://github.com/tue-robotics/kaldi_srilm/blob/master/srilm.tgz?raw=true -O srilm.tgz + extras/install_srilm.sh &> $ASR_LOG/install_srilm.log + install_srilm_status=$(grep "Installation of SRILM finished successfully" $ASR_LOG/install_srilm.log) + + if [ -z "$install_srilm_status" ] + then + echo -e "\e[34m\e[1m Install kaldi/tools/extras/install_srilm.sh failed \e[0m" + return 1 + fi + echo " - Built SRILM" # Build the src directory cd $KALDI/src @@ -100,6 +122,7 @@ _kaldi_build() echo -e "\e[34m\e[1m Configure kaldi/src failed \e[0m" return 1 fi + echo " - Configured src for build" make depend -j 8 > /dev/null make -j 8 &> $ASR_LOG/make_src.log @@ -110,6 +133,7 @@ _kaldi_build() echo -e "\e[34m\e[1m Make kaldi/src failed \e[0m" return 1 fi + echo " - Built src" # Create a STATUS file to monitor installation echo -e "\e[36m\e[1m Kaldi installation complete \e[0m" @@ -149,7 +173,7 @@ kaldi_update() kaldi_install else # Read STATUS file. If "ALL OK" then update else remove directory $KALDI - # and re-install Kaldi + # and re-install Kaldi kaldi_install_status="$(cat $KALDI/STATUS)" if [ "$kaldi_install_status" = "ALL OK" ] From fa4c53bc9e5118eca7790a95b5ade404acb74384 Mon Sep 17 00:00:00 2001 From: Arpit Aggarwal Date: Sat, 2 Feb 2019 12:10:33 +0100 Subject: [PATCH 12/60] corrected gpp_version_num extraction --- install.bash | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/install.bash b/install.bash index 9f710faf1df..6f916525ce2 100755 --- a/install.bash +++ b/install.bash @@ -43,7 +43,7 @@ _kaldi_install_dependencies() echo "Checking and installing dependencies..." # Change exit to return to source check_dependencies and change back once done sed -i "s|exit|return|g" $KALDI/tools/extras/check_dependencies.sh - source $KALDI/tools/extras/check_dependencies.sh > /dev/null 2>&1 + source $KALDI/tools/extras/check_dependencies.sh > /dev/null sed -i "s|return|exit|g" $KALDI/tools/extras/check_dependencies.sh # Install dependencies @@ -56,8 +56,14 @@ _kaldi_build() { _kaldi_install_dependencies + # if environment variable CXX is unset, set it + if [ -z "$CXX" ] + then + export CXX=g++ + fi + # Check g++ version before starting build - gpp_version_num=$(g++ --version | grep ^g++ | sed 's/^.* //g' | sed 's/\./ /g' | xargs printf "%d%02d%02d") + gpp_version_num=$($CXX -dumpversion | sed 's/\./ /g' | xargs printf "%d%02d%02d") if [ $gpp_version_num -gt 70000 ] then echo -e "\e[34m\e[1m Unsupported g++ version. Use g++ < 7.0.* \e[0m" From 91b5d4467a4f7e0c30ffd62e564608f57c21c406 Mon Sep 17 00:00:00 2001 From: Arpit Aggarwal Date: Sat, 2 Feb 2019 12:16:30 +0100 Subject: [PATCH 13/60] isolated install dependencies from build --- install.bash | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/install.bash b/install.bash index 6f916525ce2..50582b7641a 100755 --- a/install.bash +++ b/install.bash @@ -51,11 +51,19 @@ _kaldi_install_dependencies() sudo ln -s -f bash /bin/sh } +# Kaldi check dependencies +_kaldi_check_dependencies() +{ + echo "Checking dependencies..." + # Change exit to return to source check_dependencies and change back once done + sed -i "s|exit|return|g" $KALDI/tools/extras/check_dependencies.sh + $KALDI/tools/extras/check_dependencies.sh > /dev/null + sed -i "s|return|exit|g" $KALDI/tools/extras/check_dependencies.sh +} + # Kaldi Build (Common to Installation and Update) _kaldi_build() { - _kaldi_install_dependencies - # if environment variable CXX is unset, set it if [ -z "$CXX" ] then @@ -157,6 +165,7 @@ kaldi_install() # Clone repository into $KALDI echo -e "No existing installation found. Cloning from GitHub repository" git clone $KALDI_REPO $KALDI + _kaldi_install_dependencies _kaldi_build else # Read STATUS file. If not "ALL OK", remove directory $KALDI and re-install Kaldi @@ -192,6 +201,7 @@ kaldi_update() # Clean existing make _kaldi_clean # Build toolkit + _kaldi_install_dependencies _kaldi_build echo -e "\e[36m\e[1m Kaldi-ASR update complete \e[0m" @@ -235,6 +245,7 @@ else _kaldi_clean ;; --build ) + _kaldi_check_dependencies _kaldi_build ;; -h | --help ) From 49d5fa6217363883994f68a46474596ac050c501 Mon Sep 17 00:00:00 2001 From: Arpit Aggarwal Date: Sat, 2 Feb 2019 12:19:35 +0100 Subject: [PATCH 14/60] ignore install logs --- .gitignore | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.gitignore b/.gitignore index cdcd13ec8b5..d70be109e2f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,7 @@ +# TechUnited begin +/log/ +# TechUnited end + # Compiled extensionless executable files in /src/*/ # This stanza must precede wildcard patterns below! /src/*/* From 6c0a1ed3a0c636e281567d11c7cf116349c7dd52 Mon Sep 17 00:00:00 2001 From: Arpit Aggarwal Date: Sat, 2 Feb 2019 13:26:57 +0100 Subject: [PATCH 15/60] added subroutine to build gst --- install.bash | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/install.bash b/install.bash index 50582b7641a..a376b36282d 100755 --- a/install.bash +++ b/install.bash @@ -155,6 +155,34 @@ _kaldi_build() echo "ALL OK" > STATUS } +# Kaldi Gstreamer plugin with online decoder build +_kaldi_online_gst() +{ + echo "Building online decoder and gstreamer plugin..." + # Install PortAudio + cd $KALDI/tools + extras/install_portaudio.sh &> $ASR_LOG/install_portaudio.log + install_portaudio_status=$(grep "PortAudio was successfully installed" $ASR_LOG/install_portaudio.log) + + if [ -z "$install_portaudio_status" ] + then + echo -e "\e[34m\e[1m Install kaldi/tools/extras/install_portaudio.sh failed \e[0m" + return 1 + fi + echo " - Installed PortAudio" + + # Build online decoder + cd $KALDI/src/online + make -j 8 &> $ASR_LOG/make_online.log || echo -e "\e[34m\e[1m Make kaldi/src/online failed \e[0m"; return 1 + echo " - Built online decoder" + + # Build Gstreamer plugin + cd $KALDI/src/gst-plugin + make depend -j 8 > /dev/null + make -j 8 &> $ASR_LOG/make_gst-plugin.log || echo -e "\e[34m\e[1m Make kaldi/src/gst-plugin failed \e[0m"; return 1 + echo " - Built gstreamer plugin" +} + # Kaldi Installation kaldi_install() { From 8a3169788d0e3a4eac9ada43eaaa99a9f3f84e3e Mon Sep 17 00:00:00 2001 From: Arpit Aggarwal Date: Sat, 2 Feb 2019 14:15:31 +0100 Subject: [PATCH 16/60] changed build flag to tue and added online gst --- install.bash | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/install.bash b/install.bash index a376b36282d..f4bd3b4c9ac 100755 --- a/install.bash +++ b/install.bash @@ -150,7 +150,6 @@ _kaldi_build() echo " - Built src" # Create a STATUS file to monitor installation - echo -e "\e[36m\e[1m Kaldi installation complete \e[0m" cd $KALDI echo "ALL OK" > STATUS } @@ -264,7 +263,8 @@ else do case $1 in --install-kaldi ) - kaldi_install ;; + kaldi_install + echo -e "\e[36m\e[1m Kaldi installation complete \e[0m" ;; --update-kaldi ) kaldi_update ;; @@ -272,9 +272,11 @@ else --clean ) _kaldi_clean ;; - --build ) + --tue ) _kaldi_check_dependencies - _kaldi_build ;; + _kaldi_build + _kaldi_online_gst + echo -e "\e[36m\e[1m Kaldi installation complete \e[0m" ;; -h | --help ) usage From 8cd0c6e2f5ad7125454c139dc281c08508e9a26f Mon Sep 17 00:00:00 2001 From: Arpit Aggarwal Date: Sat, 2 Feb 2019 14:27:20 +0100 Subject: [PATCH 17/60] added gst-plugin to GST_PLUGIN_PATH --- setup.bash | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup.bash b/setup.bash index ded6ca17aac..8ccbd403e78 100755 --- a/setup.bash +++ b/setup.bash @@ -1,5 +1,6 @@ #! /usr/bin/env bash # -# Environment variables to use LASeR +# Environment variables to use by speech_recognition repo export KALDI_ROOT="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +export GST_PLUGIN_PATH=$KALDI_ROOT/src/gst-plugin${GST_PLUGIN_PATH:+:${GST_PLUGIN_PATH}} From 9e6ec3fca4055cab14852cf318ce901677029fe3 Mon Sep 17 00:00:00 2001 From: Arpit Aggarwal Date: Sat, 2 Feb 2019 14:33:51 +0100 Subject: [PATCH 18/60] added gstreamer plugin test --- install.bash | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/install.bash b/install.bash index f4bd3b4c9ac..e02a27bdcc8 100755 --- a/install.bash +++ b/install.bash @@ -180,6 +180,11 @@ _kaldi_online_gst() make depend -j 8 > /dev/null make -j 8 &> $ASR_LOG/make_gst-plugin.log || echo -e "\e[34m\e[1m Make kaldi/src/gst-plugin failed \e[0m"; return 1 echo " - Built gstreamer plugin" + + # Test Gstreamer plugin + export GST_PLUGIN_PATH=$KALDI/src/gst-plugin${GST_PLUGIN_PATH:+:${GST_PLUGIN_PATH}} + gst-inspect-1.0 onlinegmmdecodefaster > /dev/null || echo -e "\e[34m\e[1m gst-inspect of onlinegmmdecodefaster failed \e[0m"; return 1 + echo " - Gstreamer Plugin Test Successful" } # Kaldi Installation From abb1016768661b20718b16f228ce3381a2ef3933 Mon Sep 17 00:00:00 2001 From: Arpit Aggarwal Date: Sat, 2 Feb 2019 14:39:59 +0100 Subject: [PATCH 19/60] updated usage and dependencies --- install.bash | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/install.bash b/install.bash index e02a27bdcc8..eeccfc4c78f 100755 --- a/install.bash +++ b/install.bash @@ -14,9 +14,9 @@ usage() echo "Usage: sudo ./setup [options] " echo -e "Options:\n \ -h | --help\n \ - --build\n \ - --install\n \ - --update\n \ + --tue\n \ + --install-kaldi\n \ + --update-kaldi\n \ --clean" echo echo "-----------------------------------------------------------------------------" @@ -35,7 +35,7 @@ fi # Install the required packages and dependencies sudo apt-get update -qq > /dev/null 2>&1 sudo apt-get upgrade --assume-yes -qq > /dev/null 2>&1 -sudo apt-get install --assume-yes build-essential git python python-scipy sox swig zip -qq > /dev/null 2>&1 +sudo apt-get install --assume-yes build-essential git python python-numpy sox swig zip -qq > /dev/null 2>&1 # Kaldi install dependencies _kaldi_install_dependencies() From f243cff3cff627b0029dcc7e53bdc458cf4949fb Mon Sep 17 00:00:00 2001 From: Arpit Aggarwal Date: Sat, 2 Feb 2019 16:29:49 +0100 Subject: [PATCH 20/60] removed exit-return swap in check dependencies --- install.bash | 2 -- 1 file changed, 2 deletions(-) diff --git a/install.bash b/install.bash index eeccfc4c78f..93704701d9b 100755 --- a/install.bash +++ b/install.bash @@ -56,9 +56,7 @@ _kaldi_check_dependencies() { echo "Checking dependencies..." # Change exit to return to source check_dependencies and change back once done - sed -i "s|exit|return|g" $KALDI/tools/extras/check_dependencies.sh $KALDI/tools/extras/check_dependencies.sh > /dev/null - sed -i "s|return|exit|g" $KALDI/tools/extras/check_dependencies.sh } # Kaldi Build (Common to Installation and Update) From e55b6f78f310504699a60e8d95f32c3ab6595f79 Mon Sep 17 00:00:00 2001 From: Arpit Aggarwal Date: Sat, 2 Feb 2019 16:31:05 +0100 Subject: [PATCH 21/60] added return on failure --- install.bash | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/install.bash b/install.bash index 93704701d9b..3bb85bfd24e 100755 --- a/install.bash +++ b/install.bash @@ -276,9 +276,9 @@ else _kaldi_clean ;; --tue ) - _kaldi_check_dependencies - _kaldi_build - _kaldi_online_gst + _kaldi_check_dependencies || return 1 + _kaldi_build || return 1 + _kaldi_online_gst || return 1 echo -e "\e[36m\e[1m Kaldi installation complete \e[0m" ;; -h | --help ) From f7f1096857de748a873be998c1e7465c8a651aaf Mon Sep 17 00:00:00 2001 From: Arpit Aggarwal Date: Tue, 12 Feb 2019 20:02:26 +0100 Subject: [PATCH 22/60] rm irstlm built dir and download srilm if not existing --- install.bash | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/install.bash b/install.bash index 3bb85bfd24e..dc233030dd2 100755 --- a/install.bash +++ b/install.bash @@ -91,6 +91,11 @@ _kaldi_build() echo " - Built tools" # Install IRSTLM + # Remove IRSTLM dir if it exists else the install script fails + if [ -d "irstlm" ] + then + rm -rf irstlm + fi extras/install_irstlm.sh &> $ASR_LOG/install_irstlm.log install_irstlm_status=$(grep "Installation of IRSTLM finished successfully" $ASR_LOG/install_irstlm.log ) @@ -113,7 +118,11 @@ _kaldi_build() echo " - Built SEQUITUR" # Install SRILM - wget https://github.com/tue-robotics/kaldi_srilm/blob/master/srilm.tgz?raw=true -O srilm.tgz + # Download SRILM if .tgz does not exist + if [ ! -f "srilm.tgz" ] + then + wget https://github.com/tue-robotics/kaldi_srilm/blob/master/srilm.tgz?raw=true -O srilm.tgz + fi extras/install_srilm.sh &> $ASR_LOG/install_srilm.log install_srilm_status=$(grep "Installation of SRILM finished successfully" $ASR_LOG/install_srilm.log) From 7413f6c55442b380eacaaf7b88418d5ad4cc25fe Mon Sep 17 00:00:00 2001 From: Arpit Aggarwal Date: Tue, 12 Feb 2019 20:08:55 +0100 Subject: [PATCH 23/60] changed return to exit --- install.bash | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/install.bash b/install.bash index dc233030dd2..e52d26e9252 100755 --- a/install.bash +++ b/install.bash @@ -285,9 +285,9 @@ else _kaldi_clean ;; --tue ) - _kaldi_check_dependencies || return 1 - _kaldi_build || return 1 - _kaldi_online_gst || return 1 + _kaldi_check_dependencies || exit 1 + _kaldi_build || exit 1 + _kaldi_online_gst || exit 1 echo -e "\e[36m\e[1m Kaldi installation complete \e[0m" ;; -h | --help ) From 2a722913ecc100d7f01a5cb68dc80322d91d9849 Mon Sep 17 00:00:00 2001 From: Arpit Aggarwal Date: Wed, 20 Feb 2019 10:00:29 +0100 Subject: [PATCH 24/60] added tue-get kaldi installation --- Dockerfile | 43 +++++++------------------------------------ 1 file changed, 7 insertions(+), 36 deletions(-) diff --git a/Dockerfile b/Dockerfile index f6af8ecc9aa..4b7434c8a35 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,44 +1,15 @@ -# ---------------------------------------------------------- -# Dockerfile to build base working Ubuntu image -# ---------------------------------------------------------- +# ----------------------------------------------------------------------------- +# Dockerfile to build Kaldi gstreamer server with tue-env +# ----------------------------------------------------------------------------- -# Set the base image to Ubuntu 16.04 -FROM ubuntu:16.04 +# Set the base image to Ubuntu with tue-env +FROM tueroboticsamigo/tue-env:master # File Author / Maintainer MAINTAINER Arpit Aggarwal -# Make image to be noninteractive to use with CI -# Comment this line when not using this image with CI -ENV DEBIAN_FRONTEND=noninteractive - -# Set environment variable USER -ENV USER=hero - # Update the image and install basic packages -RUN apt-get update -qq > /dev/null && apt-get install --assume-yes --no-install-recommends apt-utils ca-certificates curl dbus dialog git sudo wget -qq > /dev/null - -# Add user USER with SUDO previleges and disable SUDO password -RUN addgroup -gid 2000 $USER && \ - adduser --disabled-password --gecos "" -uid 2000 -gid 2000 $USER && \ - addgroup $USER adm && \ - addgroup $USER sudo && \ - echo "%sudo ALL=(ALL) NOPASSWD:ALL" > /etc/sudoers.d/"$USER" - -USER "$USER" - -# Set WORKDIR to home of arpit -WORKDIR /home/"$USER" - -# Copy contents of repository into directory _git in the home of the user -COPY / ./_git/ - -# Change owner of directory to USER from root -RUN sudo chown -R $USER:$USER _git - -# Run setup and install all dependencies -RUN sudo ./_git/install.bash --complete +RUN sudo apt-get update -qq && \ + tue-get install kaldi && \ -# Start container of the Docker image at bash prompt -CMD ["/bin/bash"] From 3173d31a636a58f87c04f1d69a83137c2f737280 Mon Sep 17 00:00:00 2001 From: Loy van Beek Date: Tue, 26 Feb 2019 20:34:36 +0100 Subject: [PATCH 25/60] Fix '[WARNING]: Empty continuation line found in: ...' --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 4b7434c8a35..1d10b1b1142 100644 --- a/Dockerfile +++ b/Dockerfile @@ -10,6 +10,6 @@ MAINTAINER Arpit Aggarwal # Update the image and install basic packages RUN sudo apt-get update -qq && \ - tue-get install kaldi && \ + tue-get install kaldi From 146183cf3adf100d20fcf14db0e7c09fb835ddeb Mon Sep 17 00:00:00 2001 From: Arpit Aggarwal Date: Wed, 27 Feb 2019 10:14:44 +0100 Subject: [PATCH 26/60] removed deprecated command --- Dockerfile | 3 --- 1 file changed, 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 1d10b1b1142..b6c56026a59 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,9 +5,6 @@ # Set the base image to Ubuntu with tue-env FROM tueroboticsamigo/tue-env:master -# File Author / Maintainer -MAINTAINER Arpit Aggarwal - # Update the image and install basic packages RUN sudo apt-get update -qq && \ tue-get install kaldi From 4a61221de898afe023e2125922dedebef50f0e7a Mon Sep 17 00:00:00 2001 From: Arpit Aggarwal Date: Wed, 27 Feb 2019 10:18:10 +0100 Subject: [PATCH 27/60] made tue-env tools available to intermediate image --- Dockerfile | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index b6c56026a59..008ba9b824b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -7,6 +7,9 @@ FROM tueroboticsamigo/tue-env:master # Update the image and install basic packages RUN sudo apt-get update -qq && \ + # Make tue-env available to the intermediate image + # This step needs to be executed at every RUN step + source ~/.bashrc && \ + # Install kaldi tue-get install kaldi - From 1060c83fe4ec2f1a5e2200d871a51270169b9c43 Mon Sep 17 00:00:00 2001 From: Arpit Aggarwal Date: Mon, 4 Mar 2019 00:14:05 +0100 Subject: [PATCH 28/60] update base image and targets repo branch --- Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 008ba9b824b..5e99e9db7e5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,7 +3,7 @@ # ----------------------------------------------------------------------------- # Set the base image to Ubuntu with tue-env -FROM tueroboticsamigo/tue-env:master +FROM tueroboticsamigo/tue-env:cleanup_docker-and-ci # Update the image and install basic packages RUN sudo apt-get update -qq && \ @@ -11,5 +11,6 @@ RUN sudo apt-get update -qq && \ # This step needs to be executed at every RUN step source ~/.bashrc && \ # Install kaldi + git -C $TUE_ENV_TARGETS_DIR checkout update/speech-recognition-dev-targets && \ tue-get install kaldi From 33a5d5ab8a74b7641b8a360ff18ed2d9fd6e23e1 Mon Sep 17 00:00:00 2001 From: Arpit Aggarwal Date: Mon, 4 Mar 2019 00:18:54 +0100 Subject: [PATCH 29/60] cleanup --- install.bash | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/install.bash b/install.bash index e52d26e9252..c348611b4e0 100755 --- a/install.bash +++ b/install.bash @@ -34,12 +34,12 @@ fi # Install the required packages and dependencies sudo apt-get update -qq > /dev/null 2>&1 -sudo apt-get upgrade --assume-yes -qq > /dev/null 2>&1 -sudo apt-get install --assume-yes build-essential git python python-numpy sox swig zip -qq > /dev/null 2>&1 # Kaldi install dependencies _kaldi_install_dependencies() { + # Install base packages + sudo apt-get install --assume-yes build-essential git python python-numpy sox swig zip -qq > /dev/null 2>&1 echo "Checking and installing dependencies..." # Change exit to return to source check_dependencies and change back once done sed -i "s|exit|return|g" $KALDI/tools/extras/check_dependencies.sh @@ -48,7 +48,6 @@ _kaldi_install_dependencies() # Install dependencies sudo apt-get install libatlas3-base $debian_packages -qq > /dev/null 2>&1 - sudo ln -s -f bash /bin/sh } # Kaldi check dependencies From 9db65c5caaed028d2dac2b081aa72ac506076a4a Mon Sep 17 00:00:00 2001 From: Arpit Aggarwal Date: Mon, 4 Mar 2019 01:01:22 +0100 Subject: [PATCH 30/60] ignore everthing --- .dockerignore | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.dockerignore b/.dockerignore index 6bc11796359..2717ed259e3 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,5 +1,5 @@ # dockerignore file to minimalize the size of docker image # # Ignore .git and dev directories -.git -dev +/** + From cb463cd505fb7efe11c14a5513ebe94f2aa38add Mon Sep 17 00:00:00 2001 From: Arpit Aggarwal Date: Mon, 4 Mar 2019 01:28:30 +0100 Subject: [PATCH 31/60] fixed command grouping and added gst-plugin option --- install.bash | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/install.bash b/install.bash index c348611b4e0..576ef0b7965 100755 --- a/install.bash +++ b/install.bash @@ -17,6 +17,7 @@ usage() --tue\n \ --install-kaldi\n \ --update-kaldi\n \ + --gst-plugin\n \ --clean" echo echo "-----------------------------------------------------------------------------" @@ -178,18 +179,18 @@ _kaldi_online_gst() # Build online decoder cd $KALDI/src/online - make -j 8 &> $ASR_LOG/make_online.log || echo -e "\e[34m\e[1m Make kaldi/src/online failed \e[0m"; return 1 + make -j 8 &> $ASR_LOG/make_online.log || { echo -e "\e[34m\e[1m Make kaldi/src/online failed \e[0m"; return 1; } echo " - Built online decoder" # Build Gstreamer plugin cd $KALDI/src/gst-plugin make depend -j 8 > /dev/null - make -j 8 &> $ASR_LOG/make_gst-plugin.log || echo -e "\e[34m\e[1m Make kaldi/src/gst-plugin failed \e[0m"; return 1 + make -j 8 &> $ASR_LOG/make_gst-plugin.log || { echo -e "\e[34m\e[1m Make kaldi/src/gst-plugin failed \e[0m"; return 1; } echo " - Built gstreamer plugin" # Test Gstreamer plugin export GST_PLUGIN_PATH=$KALDI/src/gst-plugin${GST_PLUGIN_PATH:+:${GST_PLUGIN_PATH}} - gst-inspect-1.0 onlinegmmdecodefaster > /dev/null || echo -e "\e[34m\e[1m gst-inspect of onlinegmmdecodefaster failed \e[0m"; return 1 + gst-inspect-1.0 onlinegmmdecodefaster > /dev/null || { echo -e "\e[34m\e[1m gst-inspect of onlinegmmdecodefaster failed \e[0m"; return 1; } echo " - Gstreamer Plugin Test Successful" } @@ -283,6 +284,9 @@ else --clean ) _kaldi_clean ;; + --gst-plugin ) + _kaldi_online_gst ;; + --tue ) _kaldi_check_dependencies || exit 1 _kaldi_build || exit 1 From cf184f357bf468a9009e958d2de23ddecefa7c59 Mon Sep 17 00:00:00 2001 From: Arpit Aggarwal Date: Mon, 4 Mar 2019 01:32:28 +0100 Subject: [PATCH 32/60] removed targets checkout --- Dockerfile | 1 - 1 file changed, 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 5e99e9db7e5..29b2bad3243 100644 --- a/Dockerfile +++ b/Dockerfile @@ -11,6 +11,5 @@ RUN sudo apt-get update -qq && \ # This step needs to be executed at every RUN step source ~/.bashrc && \ # Install kaldi - git -C $TUE_ENV_TARGETS_DIR checkout update/speech-recognition-dev-targets && \ tue-get install kaldi From fe7443335994d0bcb07786a2bbd59d393c4bb990 Mon Sep 17 00:00:00 2001 From: Arpit Aggarwal Date: Mon, 4 Mar 2019 15:32:51 +0100 Subject: [PATCH 33/60] added comment to change image tag to master after PR --- Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Dockerfile b/Dockerfile index 29b2bad3243..8a9e024683d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,6 +3,8 @@ # ----------------------------------------------------------------------------- # Set the base image to Ubuntu with tue-env +# TODO: Change image tag to master after +# https://github.com/tue-robotics/tue-env/pull/368 is merged FROM tueroboticsamigo/tue-env:cleanup_docker-and-ci # Update the image and install basic packages From cb7caee8edab5cfd7b9751059c630013d0db11c5 Mon Sep 17 00:00:00 2001 From: Arpit Aggarwal Date: Tue, 5 Mar 2019 11:34:43 +0100 Subject: [PATCH 34/60] made wget quiet --- install.bash | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install.bash b/install.bash index 576ef0b7965..8bee22d921f 100755 --- a/install.bash +++ b/install.bash @@ -121,7 +121,7 @@ _kaldi_build() # Download SRILM if .tgz does not exist if [ ! -f "srilm.tgz" ] then - wget https://github.com/tue-robotics/kaldi_srilm/blob/master/srilm.tgz?raw=true -O srilm.tgz + wget -q https://github.com/tue-robotics/kaldi_srilm/blob/master/srilm.tgz?raw=true -O srilm.tgz fi extras/install_srilm.sh &> $ASR_LOG/install_srilm.log install_srilm_status=$(grep "Installation of SRILM finished successfully" $ASR_LOG/install_srilm.log) From 07ddba13b6b28d620fb71787c23a0ca8e992d85b Mon Sep 17 00:00:00 2001 From: Arpit Aggarwal Date: Tue, 12 Mar 2019 19:53:07 +0100 Subject: [PATCH 35/60] updated base image --- Dockerfile | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 8a9e024683d..be96d3294cf 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,9 +3,7 @@ # ----------------------------------------------------------------------------- # Set the base image to Ubuntu with tue-env -# TODO: Change image tag to master after -# https://github.com/tue-robotics/tue-env/pull/368 is merged -FROM tueroboticsamigo/tue-env:cleanup_docker-and-ci +FROM tuerobotics/tue-env:master # Update the image and install basic packages RUN sudo apt-get update -qq && \ From f9f8f8661e44373d1cf359133abe483906462a05 Mon Sep 17 00:00:00 2001 From: Arpit Aggarwal Date: Tue, 2 Apr 2019 09:53:21 +0200 Subject: [PATCH 36/60] added fast compilation --- install.bash | 2 ++ 1 file changed, 2 insertions(+) diff --git a/install.bash b/install.bash index 8bee22d921f..3e9231f2ca3 100755 --- a/install.bash +++ b/install.bash @@ -145,6 +145,8 @@ _kaldi_build() fi echo " - Configured src for build" + # Make Kaldi without checks (ensures faster compilation) + sed -i '/-g # -O0 -DKALDI_PARANOID/c\-O3 -DNDEBUG' kaldi.mk make depend -j 8 > /dev/null make -j 8 &> $ASR_LOG/make_src.log make_src_status=$( grep "Done" $ASR_LOG/make_src.log ) From 593ff56b6ad8db0d7d889b1c583381ee837c037f Mon Sep 17 00:00:00 2001 From: Arpit Aggarwal Date: Tue, 2 Apr 2019 10:02:19 +0200 Subject: [PATCH 37/60] moved g++ version check to targets repo --- install.bash | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/install.bash b/install.bash index 3e9231f2ca3..57a0a5d94cd 100755 --- a/install.bash +++ b/install.bash @@ -62,20 +62,6 @@ _kaldi_check_dependencies() # Kaldi Build (Common to Installation and Update) _kaldi_build() { - # if environment variable CXX is unset, set it - if [ -z "$CXX" ] - then - export CXX=g++ - fi - - # Check g++ version before starting build - gpp_version_num=$($CXX -dumpversion | sed 's/\./ /g' | xargs printf "%d%02d%02d") - if [ $gpp_version_num -gt 70000 ] - then - echo -e "\e[34m\e[1m Unsupported g++ version. Use g++ < 7.0.* \e[0m" - return 1 - fi - # Build toolkit echo "Building toolkit..." # Build the tools directory From 19d6f0282ebc0bb901ffc33f16a67e5bb2e24551 Mon Sep 17 00:00:00 2001 From: Arpit Aggarwal Date: Sat, 6 Apr 2019 11:11:11 +0200 Subject: [PATCH 38/60] fix duplicate property warnings --- src/gst-plugin/gst-online-gmm-decode-faster.cc | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/gst-plugin/gst-online-gmm-decode-faster.cc b/src/gst-plugin/gst-online-gmm-decode-faster.cc index 958bce41d80..3d3f6ee2f37 100644 --- a/src/gst-plugin/gst-online-gmm-decode-faster.cc +++ b/src/gst-plugin/gst-online-gmm-decode-faster.cc @@ -284,6 +284,7 @@ gst_online_gmm_decode_faster_init(GstOnlineGmmDecodeFaster * filter) { // init properties from various Kaldi Opts GstElementClass * klass = GST_ELEMENT_GET_CLASS(filter); + std::set seen_options; std::vector > option_info_list; option_info_list = filter->simple_options_->GetOptionInfoList(); int32 i = 0; @@ -293,6 +294,13 @@ gst_online_gmm_decode_faster_init(GstOnlineGmmDecodeFaster * filter) { std::pair result = (*dx); SimpleOptions::OptionInfo option_info = result.second; std::string name = result.first; + + // GetOptionInfoList returns duplicate options + if (seen_options.find(name) != seen_options.end()) + continue; + + seen_options.insert(name); + switch (option_info.type) { case SimpleOptions::kBool: filter->simple_options_->GetOption(name, &tmp_bool); From ff3f6ad99594ff723eeda07612be55947f4c0afe Mon Sep 17 00:00:00 2001 From: Arpit Aggarwal Date: Sat, 6 Apr 2019 11:19:17 +0200 Subject: [PATCH 39/60] ignore kaldimarshal files --- .gitignore | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/.gitignore b/.gitignore index d70be109e2f..659da27af93 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,3 @@ -# TechUnited begin -/log/ -# TechUnited end - # Compiled extensionless executable files in /src/*/ # This stanza must precede wildcard patterns below! /src/*/* @@ -11,6 +7,12 @@ !/src/*/Makefile !/src/*/README +# TechUnited begin +/log/ +/src/gst-plugin/kaldimarshal.cc +/src/gst-plugin/kaldimarshal.h +# TechUnited end + # Compiled Object files and python ciles *.slo *.lo From d67289271d9681b151f5bc7002b90f3700590405 Mon Sep 17 00:00:00 2001 From: Arpit Aggarwal Date: Sat, 6 Apr 2019 11:28:50 +0200 Subject: [PATCH 40/60] Revert "fix duplicate property warnings" This reverts commit 19d6f0282ebc0bb901ffc33f16a67e5bb2e24551. --- src/gst-plugin/gst-online-gmm-decode-faster.cc | 8 -------- 1 file changed, 8 deletions(-) diff --git a/src/gst-plugin/gst-online-gmm-decode-faster.cc b/src/gst-plugin/gst-online-gmm-decode-faster.cc index 3d3f6ee2f37..958bce41d80 100644 --- a/src/gst-plugin/gst-online-gmm-decode-faster.cc +++ b/src/gst-plugin/gst-online-gmm-decode-faster.cc @@ -284,7 +284,6 @@ gst_online_gmm_decode_faster_init(GstOnlineGmmDecodeFaster * filter) { // init properties from various Kaldi Opts GstElementClass * klass = GST_ELEMENT_GET_CLASS(filter); - std::set seen_options; std::vector > option_info_list; option_info_list = filter->simple_options_->GetOptionInfoList(); int32 i = 0; @@ -294,13 +293,6 @@ gst_online_gmm_decode_faster_init(GstOnlineGmmDecodeFaster * filter) { std::pair result = (*dx); SimpleOptions::OptionInfo option_info = result.second; std::string name = result.first; - - // GetOptionInfoList returns duplicate options - if (seen_options.find(name) != seen_options.end()) - continue; - - seen_options.insert(name); - switch (option_info.type) { case SimpleOptions::kBool: filter->simple_options_->GetOption(name, &tmp_bool); From 84b785b0b6c84ae2a803c3601e17039dfe351356 Mon Sep 17 00:00:00 2001 From: Arpit Aggarwal Date: Sun, 7 Apr 2019 12:33:31 +0200 Subject: [PATCH 41/60] added kaldi tools to PATH --- setup.bash | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/setup.bash b/setup.bash index 8ccbd403e78..22c90d399d6 100755 --- a/setup.bash +++ b/setup.bash @@ -4,3 +4,17 @@ export KALDI_ROOT="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" export GST_PLUGIN_PATH=$KALDI_ROOT/src/gst-plugin${GST_PLUGIN_PATH:+:${GST_PLUGIN_PATH}} + +[ -f $KALDI_ROOT/tools/env.sh ] && . $KALDI_ROOT/tools/env.sh + +export PATH=$KALDI_ROOT/egs/wsj/s5/utils/:$KALDI_ROOT/tools/openfst/bin:$PATH + +[ ! -f $KALDI_ROOT/tools/config/common_path.sh ] && echo >&2 "The standard file $KALDI_ROOT/tools/config/common_path.sh is not present -> Exit!" && exit 1 +. $KALDI_ROOT/tools/config/common_path.sh + +# Make sure that MITLM shared libs are found by the dynamic linker/loader +#export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$(pwd)/tools/mitlm-svn/lib + +# Needed for "correct" sorting +export LC_ALL=C + From 568812a1297a1c0012d50e33e09ef2f646512d8e Mon Sep 17 00:00:00 2001 From: Arpit Aggarwal Date: Sun, 7 Apr 2019 12:36:16 +0200 Subject: [PATCH 42/60] improved syntax of exports --- setup.bash | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/setup.bash b/setup.bash index 22c90d399d6..94579ad0cf7 100755 --- a/setup.bash +++ b/setup.bash @@ -2,12 +2,16 @@ # # Environment variables to use by speech_recognition repo -export KALDI_ROOT="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -export GST_PLUGIN_PATH=$KALDI_ROOT/src/gst-plugin${GST_PLUGIN_PATH:+:${GST_PLUGIN_PATH}} +KALDI_ROOT="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +export KALDI_ROOT + +GST_PLUGIN_PATH=$KALDI_ROOT/src/gst-plugin${GST_PLUGIN_PATH:+:${GST_PLUGIN_PATH}} +export GST_PLUGIN_PATH [ -f $KALDI_ROOT/tools/env.sh ] && . $KALDI_ROOT/tools/env.sh -export PATH=$KALDI_ROOT/egs/wsj/s5/utils/:$KALDI_ROOT/tools/openfst/bin:$PATH +PATH=$KALDI_ROOT/egs/wsj/s5/utils/:$KALDI_ROOT/tools/openfst/bin${PATH:+:${PATH}} +export PATH [ ! -f $KALDI_ROOT/tools/config/common_path.sh ] && echo >&2 "The standard file $KALDI_ROOT/tools/config/common_path.sh is not present -> Exit!" && exit 1 . $KALDI_ROOT/tools/config/common_path.sh @@ -16,5 +20,6 @@ export PATH=$KALDI_ROOT/egs/wsj/s5/utils/:$KALDI_ROOT/tools/openfst/bin:$PATH #export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$(pwd)/tools/mitlm-svn/lib # Needed for "correct" sorting -export LC_ALL=C +LC_ALL=C +export LC_ALL From f1cdd3beaf0b23de40cd57ac1cfd10646df189fa Mon Sep 17 00:00:00 2001 From: Arpit Aggarwal Date: Tue, 5 Nov 2019 20:54:05 +0100 Subject: [PATCH 43/60] [setup] Removed gstreamer and cleaned up exports --- setup.bash | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/setup.bash b/setup.bash index 94579ad0cf7..57bb3aed0c4 100755 --- a/setup.bash +++ b/setup.bash @@ -2,24 +2,14 @@ # # Environment variables to use by speech_recognition repo -KALDI_ROOT="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -export KALDI_ROOT - -GST_PLUGIN_PATH=$KALDI_ROOT/src/gst-plugin${GST_PLUGIN_PATH:+:${GST_PLUGIN_PATH}} -export GST_PLUGIN_PATH +export KALDI_ROOT="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" [ -f $KALDI_ROOT/tools/env.sh ] && . $KALDI_ROOT/tools/env.sh -PATH=$KALDI_ROOT/egs/wsj/s5/utils/:$KALDI_ROOT/tools/openfst/bin${PATH:+:${PATH}} -export PATH +export PATH=$KALDI_ROOT/egs/wsj/s5/utils/:$KALDI_ROOT/tools/openfst/bin${PATH:+:${PATH}} [ ! -f $KALDI_ROOT/tools/config/common_path.sh ] && echo >&2 "The standard file $KALDI_ROOT/tools/config/common_path.sh is not present -> Exit!" && exit 1 . $KALDI_ROOT/tools/config/common_path.sh # Make sure that MITLM shared libs are found by the dynamic linker/loader #export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$(pwd)/tools/mitlm-svn/lib - -# Needed for "correct" sorting -LC_ALL=C -export LC_ALL - From 59dd718483ebabf768fbe7349ab1e0afaba70363 Mon Sep 17 00:00:00 2001 From: Arpit Aggarwal Date: Tue, 19 Nov 2019 19:19:16 +0100 Subject: [PATCH 44/60] [install] Removed build of gstreamer from tue command --- install.bash | 1 - 1 file changed, 1 deletion(-) diff --git a/install.bash b/install.bash index 57a0a5d94cd..ea1e5b3e38e 100755 --- a/install.bash +++ b/install.bash @@ -278,7 +278,6 @@ else --tue ) _kaldi_check_dependencies || exit 1 _kaldi_build || exit 1 - _kaldi_online_gst || exit 1 echo -e "\e[36m\e[1m Kaldi installation complete \e[0m" ;; -h | --help ) From 1bfe5204feefed0692dd374d3c24690e5470a857 Mon Sep 17 00:00:00 2001 From: Arpit Aggarwal Date: Tue, 19 Nov 2019 19:21:49 +0100 Subject: [PATCH 45/60] [install] Updated usage doc --- install.bash | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/install.bash b/install.bash index ea1e5b3e38e..c596ce62933 100755 --- a/install.bash +++ b/install.bash @@ -9,9 +9,9 @@ usage() { echo "-----------------------------------------------------------------------------" - echo -e "\e[35m\e[1m LASeR: Linux Automatic Speech Recognition \e[0m" + echo -e "\e[35m\e[1m Kaldi Automatic Speech Recognition \e[0m" echo "-----------------------------------------------------------------------------" - echo "Usage: sudo ./setup [options] " + echo "Usage: sudo ./install.bash [options] " echo -e "Options:\n \ -h | --help\n \ --tue\n \ From f7483607f1e1776bbb307e88b8d54008661e8436 Mon Sep 17 00:00:00 2001 From: Arpit Aggarwal Date: Tue, 28 Jan 2020 22:33:30 +0100 Subject: [PATCH 46/60] feat: Split build function into smaller functions --- install.bash | 48 ++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 44 insertions(+), 4 deletions(-) diff --git a/install.bash b/install.bash index c596ce62933..c24805cf36f 100755 --- a/install.bash +++ b/install.bash @@ -55,12 +55,11 @@ _kaldi_install_dependencies() _kaldi_check_dependencies() { echo "Checking dependencies..." - # Change exit to return to source check_dependencies and change back once done $KALDI/tools/extras/check_dependencies.sh > /dev/null } # Kaldi Build (Common to Installation and Update) -_kaldi_build() +_kaldi_build_tools() { # Build toolkit echo "Building toolkit..." @@ -75,9 +74,13 @@ _kaldi_build() return 1 fi echo " - Built tools" +} +_kaldi_build_irstlm() +{ # Install IRSTLM # Remove IRSTLM dir if it exists else the install script fails + cd $KALDI/tools if [ -d "irstlm" ] then rm -rf irstlm @@ -91,8 +94,12 @@ _kaldi_build() return 1 fi echo " - Built IRSTLM" +} +_kaldi_build_sequitur() +{ # Install Sequitur + cd $KALDI/tools extras/install_sequitur.sh &> $ASR_LOG/install_sequitur.log install_sequitur_status=$(grep "Installation of SEQUITUR finished successfully" $ASR_LOG/install_sequitur.log) @@ -102,8 +109,12 @@ _kaldi_build() return 1 fi echo " - Built SEQUITUR" +} +_kaldi_build_srilm() +{ # Install SRILM + cd $KALDI/tools # Download SRILM if .tgz does not exist if [ ! -f "srilm.tgz" ] then @@ -118,7 +129,27 @@ _kaldi_build() return 1 fi echo " - Built SRILM" +} + +_kaldi_build_src_cmake() +{ + # Build src using CMake + cd $KALDI + + if [ -d build ] + then + rm -rf build + fi + + mkdir build + cd build + cmake -GNinja -DCMAKE_INSTALL_PREFIX=../dist -DKALDI_BUILD_EXE=OFF -DKALDI_BUILD_TEST=OFF -DBUILD_SHARED_LIBS=ON .. + + +} +_kaldi_build_src_make() +{ # Build the src directory cd $KALDI/src ./configure --shared &> $ASR_LOG/configure_src.log @@ -143,6 +174,15 @@ _kaldi_build() return 1 fi echo " - Built src" +} + +_kaldi_build() +{ + _kaldi_build_tools || return 1 + _kaldi_build_irstlm || return 1 + _kaldi_build_sequitur || return 1 + _kaldi_build_srilm || return 1 + _kaldi_build_src_make || return 1 # Create a STATUS file to monitor installation cd $KALDI @@ -277,7 +317,8 @@ else --tue ) _kaldi_check_dependencies || exit 1 - _kaldi_build || exit 1 + _kaldi_build_tools || exit 1 + _kaldi_build_src_cmake || exit 1 echo -e "\e[36m\e[1m Kaldi installation complete \e[0m" ;; -h | --help ) @@ -291,4 +332,3 @@ else shift done fi - From 41868d452c40f6e0c3a6b862ef37b3843bbb6839 Mon Sep 17 00:00:00 2001 From: Arpit Aggarwal Date: Tue, 28 Jan 2020 22:43:03 +0100 Subject: [PATCH 47/60] feat: Set higher version of cmake required --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 1403ae001ee..34ad2d52824 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 3.5) +cmake_minimum_required(VERSION 3.12) project(kaldi) set(CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake;${CMAKE_MODULE_PATH}") From d382ff431398351c7bf4d8db43a9a6c37c37a4e3 Mon Sep 17 00:00:00 2001 From: Arpit Aggarwal Date: Tue, 4 Feb 2020 21:56:41 +0100 Subject: [PATCH 48/60] feat: Add variables to gather target information --- CMakeLists.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 34ad2d52824..613a0cc5bc4 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,6 +1,9 @@ cmake_minimum_required(VERSION 3.12) project(kaldi) +set(GENERATED_LIBS "" CACHE INTERNAL "") +set(ALL_TARGETS "" CACHE INTERNAL "") + set(CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake;${CMAKE_MODULE_PATH}") include(GNUInstallDirs) include(Utils) From 3042f6834b73ff8b4e3742ec3f08c036831fc9ef Mon Sep 17 00:00:00 2001 From: Arpit Aggarwal Date: Tue, 4 Feb 2020 21:57:22 +0100 Subject: [PATCH 49/60] feat: Add template for pkgconfig file --- cmake/kaldi.pc.in | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 cmake/kaldi.pc.in diff --git a/cmake/kaldi.pc.in b/cmake/kaldi.pc.in new file mode 100644 index 00000000000..19681794673 --- /dev/null +++ b/cmake/kaldi.pc.in @@ -0,0 +1,9 @@ +prefix=@DEST_DIR@ +libdir=${prefix}/lib +includedir=${prefix}/include + +Name: kaldi +Description: Kaldi Speech Recognition Toolkit +Version:@VERSION@ +Libs: -L${libdir} @PRIVATE_LIBS@ +Cflags: -I${includedir} From 6e1daed795735a080b1b98ab8675b95c677d6791 Mon Sep 17 00:00:00 2001 From: Arpit Aggarwal Date: Tue, 4 Feb 2020 21:59:00 +0100 Subject: [PATCH 50/60] feat: Add missing install step for cmake --- install.bash | 1 + 1 file changed, 1 insertion(+) diff --git a/install.bash b/install.bash index c24805cf36f..0bdbdeab4fb 100755 --- a/install.bash +++ b/install.bash @@ -144,6 +144,7 @@ _kaldi_build_src_cmake() mkdir build cd build cmake -GNinja -DCMAKE_INSTALL_PREFIX=../dist -DKALDI_BUILD_EXE=OFF -DKALDI_BUILD_TEST=OFF -DBUILD_SHARED_LIBS=ON .. + cmake --build . --target install } From 89fdf0d24726baaa7f4d01cea13920f688ce823a Mon Sep 17 00:00:00 2001 From: Arpit Aggarwal Date: Tue, 4 Feb 2020 21:59:58 +0100 Subject: [PATCH 51/60] feat: Add meta commands to populate GENERATED_LIBS and ALL_TARGETS --- cmake/gen_cmake_skeleton.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/cmake/gen_cmake_skeleton.py b/cmake/gen_cmake_skeleton.py index 8c393630cfb..c613a72ec39 100644 --- a/cmake/gen_cmake_skeleton.py +++ b/cmake/gen_cmake_skeleton.py @@ -115,12 +115,19 @@ def add_test_source(self, filename): def gen_code(self): ret = [] + ret.append('set(ALL_TARGETS ${ALL_TARGETS} ' + '{} CACHE INTERNAL "ALL_TARGETS")'.format(self.target_name)) if len(self.header_list) > 0: ret.append("set(PUBLIC_HEADERS") for f in self.header_list: ret.append(" " + f) ret.append(")\n") + ret.append('if (GENERATED_LIBS STREQUAL "")') + ret.append(' set(GENERATED_LIBS "{}" CACHE INTERNAL "GENERATED_LIBS")'.format(self.target_name)) + ret.append('else()') + ret.append(' set(GENERATED_LIBS "${GENERATED_LIBS};' + '{}" CACHE INTERNAL "GENERATED_LIBS")'.format(self.target_name)) + ret.append('endif()') + ret.append("add_library(" + self.target_name + " INTERFACE)") ret.append("target_include_directories(" + self.target_name + " INTERFACE ") ret.append(" $") @@ -171,6 +178,7 @@ def load_dependency_from_makefile(self, filename): def gen_code(self): ret = [] + ret.append('set(ALL_TARGETS ${ALL_TARGETS} ' + '{} CACHE INTERNAL "ALL_TARGETS")'.format(self.target_name)) if len(self.header_list) > 0: ret.append("set(PUBLIC_HEADERS") @@ -188,6 +196,13 @@ def gen_code(self): ret.append(" )") ret.append("endif()\n") + # ret.append('list(APPEND GENERATED_LIBS "{}")'.format(self.target_name)) + ret.append('if (GENERATED_LIBS STREQUAL "")') + ret.append(' set(GENERATED_LIBS "{}" CACHE INTERNAL "GENERATED_LIBS")'.format(self.target_name)) + ret.append('else()') + ret.append(' set(GENERATED_LIBS "${GENERATED_LIBS};' + '{}" CACHE INTERNAL "GENERATED_LIBS")'.format(self.target_name)) + ret.append('endif()') + ret.append("add_library(" + self.target_name) for f in self.source_list: ret.append(" " + f) From df652fe09ab0d75034ccc034c2631ecbe90cc41c Mon Sep 17 00:00:00 2001 From: Arpit Aggarwal Date: Tue, 4 Feb 2020 22:00:35 +0100 Subject: [PATCH 52/60] feat: Add generation of pkgconfig file --- CMakeLists.txt | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 613a0cc5bc4..3c72fd3740f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -223,6 +223,20 @@ configure_package_config_file( ${CMAKE_BINARY_DIR}/cmake/kaldi-config.cmake INSTALL_DESTINATION lib/cmake/kaldi ) + +message(STATUS "The list of generated libraries is: ${GENERATED_LIBS}") +set(DEST_DIR "${CMAKE_INSTALL_PREFIX}") + +foreach(LIB ${GENERATED_LIBS}) + set(PRIVATE_LIBS "${PRIVATE_LIBS} -l${LIB}") +endforeach() + +configure_package_config_file( + ${CMAKE_CURRENT_SOURCE_DIR}/cmake/kaldi.pc.in + ${CMAKE_BINARY_DIR}/pkgconfig/kaldi.pc + INSTALL_DESTINATION lib/pkgconfig +) + write_basic_package_version_file( ${CMAKE_BINARY_DIR}/cmake/kaldi-config-version.cmake VERSION ${KALDI_VERSION} From 7e3439de221254dc3c18e72eb239b5baccd97e5d Mon Sep 17 00:00:00 2001 From: Arpit Aggarwal Date: Tue, 4 Feb 2020 22:00:53 +0100 Subject: [PATCH 53/60] feat: Add install step for pkgconfig file --- CMakeLists.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 3c72fd3740f..aaad303f8f7 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -245,4 +245,7 @@ write_basic_package_version_file( install(FILES ${CMAKE_BINARY_DIR}/cmake/kaldi-config.cmake ${CMAKE_BINARY_DIR}/cmake/kaldi-config-version.cmake DESTINATION lib/cmake/kaldi ) + +install(FILES ${CMAKE_BINARY_DIR}/pkgconfig/kaldi.pc DESTINATION lib/pkgconfig) + install(EXPORT kaldi-targets DESTINATION ${CMAKE_INSTALL_PREFIX}/lib/cmake/kaldi) From 15e963c6c0e23370308d9c022b0adc9ea1667c40 Mon Sep 17 00:00:00 2001 From: Arpit Aggarwal Date: Sat, 7 Mar 2020 08:59:17 +0100 Subject: [PATCH 54/60] feat: Add fst lib and specify include dir paths --- cmake/kaldi.pc.in | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmake/kaldi.pc.in b/cmake/kaldi.pc.in index 19681794673..ad3a98e7f6f 100644 --- a/cmake/kaldi.pc.in +++ b/cmake/kaldi.pc.in @@ -5,5 +5,5 @@ includedir=${prefix}/include Name: kaldi Description: Kaldi Speech Recognition Toolkit Version:@VERSION@ -Libs: -L${libdir} @PRIVATE_LIBS@ -Cflags: -I${includedir} +Libs: -L${libdir} -lfst @PRIVATE_LIBS@ +Cflags: -I${includedir}/kaldi -I${includedir}/openfst From b17a3cf5dfc77e4426151b920e7b8dc3c46dfa65 Mon Sep 17 00:00:00 2001 From: Arpit Aggarwal Date: Sat, 7 Mar 2020 09:14:24 +0100 Subject: [PATCH 55/60] feat: Add PKG_CONFIG_PATH to setup and comment out the rest --- setup.bash | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/setup.bash b/setup.bash index 57bb3aed0c4..f842fc16cd7 100755 --- a/setup.bash +++ b/setup.bash @@ -4,12 +4,17 @@ export KALDI_ROOT="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -[ -f $KALDI_ROOT/tools/env.sh ] && . $KALDI_ROOT/tools/env.sh +if [[ :$PKG_CONFIG_PATH: != *:$KALDI_ROOT/dist/lib/pkgconfig:* ]] +then + export PKG_CONFIG_PATH=$KALDI_ROOT/dist/lib/pkgconfig${PKG_CONFIG_PATH:+:${PKG_CONFIG_PATH}} +fi -export PATH=$KALDI_ROOT/egs/wsj/s5/utils/:$KALDI_ROOT/tools/openfst/bin${PATH:+:${PATH}} +# [ -f $KALDI_ROOT/tools/env.sh ] && . $KALDI_ROOT/tools/env.sh -[ ! -f $KALDI_ROOT/tools/config/common_path.sh ] && echo >&2 "The standard file $KALDI_ROOT/tools/config/common_path.sh is not present -> Exit!" && exit 1 -. $KALDI_ROOT/tools/config/common_path.sh +# export PATH=$KALDI_ROOT/egs/wsj/s5/utils/:$KALDI_ROOT/tools/openfst/bin${PATH:+:${PATH}} + +# [ ! -f $KALDI_ROOT/tools/config/common_path.sh ] && echo >&2 "The standard file $KALDI_ROOT/tools/config/common_path.sh is not present -> Exit!" && exit 1 +# . $KALDI_ROOT/tools/config/common_path.sh # Make sure that MITLM shared libs are found by the dynamic linker/loader #export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$(pwd)/tools/mitlm-svn/lib From 8dd8e4fb1a6e7cb46f1a067b66f48c073b4cbeae Mon Sep 17 00:00:00 2001 From: Arpit Aggarwal Date: Sat, 7 Mar 2020 09:18:22 +0100 Subject: [PATCH 56/60] feat: Remove executable flag --- setup.bash | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100755 => 100644 setup.bash diff --git a/setup.bash b/setup.bash old mode 100755 new mode 100644 From afae1ef9d12609419db713480a81d192b69c7f3e Mon Sep 17 00:00:00 2001 From: Arpit Aggarwal Date: Sat, 7 Mar 2020 09:22:36 +0100 Subject: [PATCH 57/60] style: Remove extra spaces --- install.bash | 2 -- 1 file changed, 2 deletions(-) diff --git a/install.bash b/install.bash index 0bdbdeab4fb..960aa89e616 100755 --- a/install.bash +++ b/install.bash @@ -145,8 +145,6 @@ _kaldi_build_src_cmake() cd build cmake -GNinja -DCMAKE_INSTALL_PREFIX=../dist -DKALDI_BUILD_EXE=OFF -DKALDI_BUILD_TEST=OFF -DBUILD_SHARED_LIBS=ON .. cmake --build . --target install - - } _kaldi_build_src_make() From 05ddf43b4d6a4b19be6fa9553228cbbe915f6a65 Mon Sep 17 00:00:00 2001 From: Arpit Aggarwal Date: Sat, 7 Mar 2020 10:47:15 +0100 Subject: [PATCH 58/60] fix: Add STATUS file --- install.bash | 3 +++ 1 file changed, 3 insertions(+) diff --git a/install.bash b/install.bash index 960aa89e616..e59fbeedbdf 100755 --- a/install.bash +++ b/install.bash @@ -318,6 +318,9 @@ else _kaldi_check_dependencies || exit 1 _kaldi_build_tools || exit 1 _kaldi_build_src_cmake || exit 1 + # Create a STATUS file to monitor installation + cd $KALDI + echo "ALL OK" > STATUS echo -e "\e[36m\e[1m Kaldi installation complete \e[0m" ;; -h | --help ) From 22d76bebf8a35d5bb926de25f111a15177bb84f0 Mon Sep 17 00:00:00 2001 From: Arpit Aggarwal Date: Sat, 7 Mar 2020 12:56:28 +0100 Subject: [PATCH 59/60] fix: Ignore kaldi-itf from list of shared libs kaldi-itf is not generated as a shared lib but the script adds it to the list --- cmake/gen_cmake_skeleton.py | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/cmake/gen_cmake_skeleton.py b/cmake/gen_cmake_skeleton.py index c613a72ec39..d52bb1126be 100644 --- a/cmake/gen_cmake_skeleton.py +++ b/cmake/gen_cmake_skeleton.py @@ -122,11 +122,12 @@ def gen_code(self): ret.append(" " + f) ret.append(")\n") - ret.append('if (GENERATED_LIBS STREQUAL "")') - ret.append(' set(GENERATED_LIBS "{}" CACHE INTERNAL "GENERATED_LIBS")'.format(self.target_name)) - ret.append('else()') - ret.append(' set(GENERATED_LIBS "${GENERATED_LIBS};' + '{}" CACHE INTERNAL "GENERATED_LIBS")'.format(self.target_name)) - ret.append('endif()') + if self.target_name != 'kaldi-itf': + ret.append('if (GENERATED_LIBS STREQUAL "")') + ret.append(' set(GENERATED_LIBS "{}" CACHE INTERNAL "GENERATED_LIBS")'.format(self.target_name)) + ret.append('else()') + ret.append(' set(GENERATED_LIBS "${GENERATED_LIBS};' + '{}" CACHE INTERNAL "GENERATED_LIBS")'.format(self.target_name)) + ret.append('endif()') ret.append("add_library(" + self.target_name + " INTERFACE)") ret.append("target_include_directories(" + self.target_name + " INTERFACE ") @@ -196,12 +197,12 @@ def gen_code(self): ret.append(" )") ret.append("endif()\n") - # ret.append('list(APPEND GENERATED_LIBS "{}")'.format(self.target_name)) - ret.append('if (GENERATED_LIBS STREQUAL "")') - ret.append(' set(GENERATED_LIBS "{}" CACHE INTERNAL "GENERATED_LIBS")'.format(self.target_name)) - ret.append('else()') - ret.append(' set(GENERATED_LIBS "${GENERATED_LIBS};' + '{}" CACHE INTERNAL "GENERATED_LIBS")'.format(self.target_name)) - ret.append('endif()') + if self.target_name != 'kaldi-itf': + ret.append('if (GENERATED_LIBS STREQUAL "")') + ret.append(' set(GENERATED_LIBS "{}" CACHE INTERNAL "GENERATED_LIBS")'.format(self.target_name)) + ret.append('else()') + ret.append(' set(GENERATED_LIBS "${GENERATED_LIBS};' + '{}" CACHE INTERNAL "GENERATED_LIBS")'.format(self.target_name)) + ret.append('endif()') ret.append("add_library(" + self.target_name) for f in self.source_list: From b8b9c29ba21c43cc82447312900a1f945867c3ae Mon Sep 17 00:00:00 2001 From: Arpit Aggarwal Date: Sat, 7 Mar 2020 13:10:30 +0100 Subject: [PATCH 60/60] feat: Move install check file inside the function --- install.bash | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/install.bash b/install.bash index e59fbeedbdf..5094677e242 100755 --- a/install.bash +++ b/install.bash @@ -136,6 +136,13 @@ _kaldi_build_src_cmake() # Build src using CMake cd $KALDI + # Remove existing STATUS file from previous build + if [ -f STATUS ] + then + rm STATUS + fi + + # Remove existing build directory if [ -d build ] then rm -rf build @@ -144,7 +151,11 @@ _kaldi_build_src_cmake() mkdir build cd build cmake -GNinja -DCMAKE_INSTALL_PREFIX=../dist -DKALDI_BUILD_EXE=OFF -DKALDI_BUILD_TEST=OFF -DBUILD_SHARED_LIBS=ON .. - cmake --build . --target install + cmake --build . --target install || { echo "CMake build failed" && return 1; } + + # Create a STATUS file to monitor installation + cd $KALDI + echo "ALL OK" > STATUS } _kaldi_build_src_make() @@ -318,9 +329,6 @@ else _kaldi_check_dependencies || exit 1 _kaldi_build_tools || exit 1 _kaldi_build_src_cmake || exit 1 - # Create a STATUS file to monitor installation - cd $KALDI - echo "ALL OK" > STATUS echo -e "\e[36m\e[1m Kaldi installation complete \e[0m" ;; -h | --help )