From 1651073f8a71fd1677eef0412a355744c7a43f48 Mon Sep 17 00:00:00 2001 From: NoamDev <37066741+NoamDev@users.noreply.github.com> Date: Mon, 21 Sep 2020 14:06:09 +0300 Subject: [PATCH 1/5] Add windows support --- build.bat | 5 +++ build.sh | 2 +- precise.template.spec | 8 +++- setup.bat | 9 +++++ setup.py | 4 +- win-package.py | 90 +++++++++++++++++++++++++++++++++++++++++++ 6 files changed, 113 insertions(+), 5 deletions(-) create mode 100644 build.bat create mode 100644 setup.bat create mode 100644 win-package.py diff --git a/build.bat b/build.bat new file mode 100644 index 00000000..c624906e --- /dev/null +++ b/build.bat @@ -0,0 +1,5 @@ +@echo off +call setup.bat || exit /b +call .venv/Scripts/activate || exit /b +pip install pyinstaller || exit /b +python win-package.py || exit /b \ No newline at end of file diff --git a/build.sh b/build.sh index 5de481b0..c820e412 100755 --- a/build.sh +++ b/build.sh @@ -30,7 +30,7 @@ package_scripts() { continue fi tmp_name=$(mktemp).spec - cat "precise.template.spec" | replace "%%SCRIPT%%" "$script" | replace "%%TRAIN_LIBS%%" "$train_libs" > "$tmp_name" + cat "precise.template.spec" | replace "%%SCRIPT%%" "$script" | replace "%%TRAIN_LIBS%%" "$train_libs" | replace "%%STRIP%%" "True" > "$tmp_name" pyinstaller -y "$tmp_name" if [ "$exe" != "$combined_folder" ]; then cp -R dist/$exe/* "dist/$combined_folder" diff --git a/precise.template.spec b/precise.template.spec index 90b0b586..bcb6d227 100644 --- a/precise.template.spec +++ b/precise.template.spec @@ -5,10 +5,11 @@ from glob import iglob from os.path import basename, dirname, abspath import os import fnmatch +from PyInstaller.utils.hooks import collect_data_files script_name = '%%SCRIPT%%' train_libs = %%TRAIN_LIBS%% -strip = True +strip = %%STRIP%% site_packages = '.venv/lib/python3.6/site-packages/' hidden_imports = ['prettyparse', 'speechpy'] binaries = [] @@ -29,11 +30,13 @@ if train_libs: ] hidden_imports += ['h5py'] +datas = collect_data_files('astor', subdir=None, include_py_files=False) + a = Analysis( [abspath('precise/scripts/{}.py'.format(script_name))], pathex=['.'], binaries=binaries, - datas=[], + datas=datas, hiddenimports=hidden_imports, hookspath=[], runtime_hooks=[], @@ -43,6 +46,7 @@ a = Analysis( cipher=block_cipher ) + for i in range(len(a.binaries)): dest, origin, kind = a.binaries[i] if '_pywrap_tensorflow_internal' in dest: diff --git a/setup.bat b/setup.bat new file mode 100644 index 00000000..e75f6303 --- /dev/null +++ b/setup.bat @@ -0,0 +1,9 @@ +@echo off +if not exist ".venv" ( + py -3.6 -m venv .venv || exit /b +) +call .venv/Scripts/activate || exit /b +pip install -e runner/ || exit /b +pip install -e . || exit /b +rem Optional, for comparison +pip install pocketsphinx || exit /b \ No newline at end of file diff --git a/setup.py b/setup.py index 44a9916b..84e6925b 100644 --- a/setup.py +++ b/setup.py @@ -71,8 +71,8 @@ }, include_package_data=True, install_requires=[ - 'numpy==1.16', - 'tensorflow>=1.13,<1.14', # Must be on piwheels + 'numpy==1.16.2', + 'tensorflow==1.13.1', # Must be on piwheels 'sonopy', 'pyaudio', 'keras<=2.1.5', diff --git a/win-package.py b/win-package.py new file mode 100644 index 00000000..e862f599 --- /dev/null +++ b/win-package.py @@ -0,0 +1,90 @@ +# This file should not be used directly but using build.bat + +import os +import os.path +from os.path import join, normpath, basename +import platform +import re +import tempfile +import shutil +import glob +import tarfile +from distutils import dir_util + +from precise import __version__ + + +import hashlib +def filemd5(fname): + hash_md5 = hashlib.md5() + with open(fname, "rb") as f: + for chunk in iter(lambda: f.read(4096), b""): + hash_md5.update(chunk) + return hash_md5.hexdigest() + +def make_tarfile(output_filename, source_dir): + with tarfile.open(output_filename, "w:gz") as tar: + tar.add(source_dir, arcname=basename(source_dir)) + +def tar_name(tar_prefix): + arch = platform.machine() + return "{tar_prefix}_{version}_win_{archname}.tar.gz".format( + tar_prefix=tar_prefix, + version=__version__, + archname = arch) + +def filecontains(path, string): + try: + f = open(path) + content = f.read() + f.close() + return string in content + except FileNotFoundError: + return False + +def package_scripts(tar_prefix, combined_folder, scripts, train_libs): + completed_file=join("dist", "completed_{}.txt".format(combined_folder)) + if not os.path.isfile(completed_file): + try: + shutil.rmtree(join("dist",combined_folder)) + except FileNotFoundError: + pass + + for script in scripts: + exe = "precise-{}".format(script.replace('_', '-')) + if filecontains(completed_file, exe): + continue + with tempfile.NamedTemporaryFile(mode = 'w',suffix='.spec', delete=False) as temp_file: + temp_path = temp_file.name + with open("precise.template.spec") as template_spec: + spec = template_spec.read() \ + .replace("%%SCRIPT%%",script) \ + .replace("%%TRAIN_LIBS%%", train_libs) \ + .replace("%%STRIP%%", "False") + temp_file.write(spec + '\n') + if os.system("pyinstaller -y {} --workpath=build/{}".format(temp_path, exe)) != 0: + raise Exception("pyinstaller error") + print(temp_path) + if exe != combined_folder: + dir_util.copy_tree(join("dist",exe), join("dist",combined_folder), update=1) + shutil.rmtree(join("dist",exe)) + shutil.rmtree(join("build",exe)) + with open(completed_file, 'a') as f: + f.write(exe + '\n') + + out_name = tar_name(tar_prefix) + make_tarfile(join('dist', out_name), join('dist', combined_folder)) + with open(join('dist', "{}.md5".format(out_name)), 'w') as md5file: + md5file.write(filemd5(join('dist', out_name))) + +def main(): + all_scripts=re.findall('(?<=precise.scripts.)[a-z_]+', open('setup.py').read()) + package_scripts("precise-all", "precise", all_scripts, "True") + package_scripts("precise-engine", "precise-engine", ["engine"], "False") + + tar_1 = join("dist",tar_name("precise-all")) + tar_2 = join("dist",tar_name("precise-engine")) + print("Wrote to {} and {}".format(tar_1, tar_2)) + +if __name__ == '__main__': + main() From 5270942f1ec366cd9c684b5281947da5889a6655 Mon Sep 17 00:00:00 2001 From: NoamDev <37066741+NoamDev@users.noreply.github.com> Date: Tue, 22 Sep 2020 15:22:20 +0300 Subject: [PATCH 2/5] Restructure add docstrings to win-package.py --- win-package.py | 155 +++++++++++++++++++++++++++++++++++-------------- 1 file changed, 112 insertions(+), 43 deletions(-) diff --git a/win-package.py b/win-package.py index e862f599..029e0353 100644 --- a/win-package.py +++ b/win-package.py @@ -13,20 +13,89 @@ from precise import __version__ - import hashlib + +def main(): + all_scripts=re.findall('(?<=precise.scripts.)[a-z_]+', open('setup.py').read()) + package_scripts("precise-all", "precise", all_scripts, True) + package_scripts("precise-engine", "precise-engine", ["engine"], False) + + tar_1 = join("dist",tar_name("precise-all")) + tar_2 = join("dist",tar_name("precise-engine")) + print("Wrote to {} and {}".format(tar_1, tar_2)) + +def package_scripts(tar_prefix, combined_folder, scripts, train_libs): + """Use pyinstaller to create EXEs for the scripts + and bundle them to a tar.gz file in dist/. + + Args: + tar_prefix (str): The prefix of the output tar.gz file. + combined_folder (str): The name of the directory in dist on which put all the files produced by pyinstaller. + + """ + completed_file=join("dist", "completed_{}.txt".format(combined_folder)) + if not os.path.isfile(completed_file): + delete_dir_if_exists(join("dist", combined_folder)) + + for script in scripts: + exe = "precise-{}".format(script.replace('_', '-')) + if filecontains(completed_file, exe): + continue + + spec_path = createSpecFile(script, train_libs) + if os.system("pyinstaller -y {} --workpath=build/{}".format(spec_path, exe)) != 0: + raise Exception("pyinstaller error") + + if exe != combined_folder: + dir_util.copy_tree(join("dist",exe), join("dist",combined_folder), update=1) + shutil.rmtree(join("dist",exe)) + shutil.rmtree(join("build",exe)) + with open(completed_file, 'a') as f: + f.write(exe + '\n') + + out_name = tar_name(tar_prefix) + make_tarfile(join('dist', out_name), join('dist', combined_folder)) + with open(join('dist', "{}.md5".format(out_name)), 'w') as md5file: + md5file.write(filemd5(join('dist', out_name))) + def filemd5(fname): + """Calculate md5 hash of a file. + + Args: + fname (str): (The path of) the file to be hashed. + + Returns: + The md5 hash of the file in a hexadecimal string. + + """ hash_md5 = hashlib.md5() with open(fname, "rb") as f: for chunk in iter(lambda: f.read(4096), b""): hash_md5.update(chunk) return hash_md5.hexdigest() -def make_tarfile(output_filename, source_dir): +def make_tarfile(source_dir, output_filename): + """Compress a directory into a gzipped tar file. + + Args: + source_dir (str): (The path of) the directory to be compressed. + output_filename (str): The tar.gz file name/path the directory should be compressed to. + + """ with tarfile.open(output_filename, "w:gz") as tar: tar.add(source_dir, arcname=basename(source_dir)) def tar_name(tar_prefix): + """Generate a name for a tar.gz file which includes the version, the os name (windows) + and the architacture. + + Args: + tar_prefix (str): The tar.gz filename will start with this value. + + Returns: + A string in the following format: "{tar_prefix}_{version}_win_{archname}.tar.gz". + + """ arch = platform.machine() return "{tar_prefix}_{version}_win_{archname}.tar.gz".format( tar_prefix=tar_prefix, @@ -34,6 +103,16 @@ def tar_name(tar_prefix): archname = arch) def filecontains(path, string): + """Check if a file contains a given phrase. + + Args: + path (str): (The path of) the file to search in. + string (str): The phrase to look for. + + Returns: + True of the given file contains the given phrase, False otherwise. + + """ try: f = open(path) content = f.read() @@ -42,49 +121,39 @@ def filecontains(path, string): except FileNotFoundError: return False -def package_scripts(tar_prefix, combined_folder, scripts, train_libs): - completed_file=join("dist", "completed_{}.txt".format(combined_folder)) - if not os.path.isfile(completed_file): - try: - shutil.rmtree(join("dist",combined_folder)) - except FileNotFoundError: - pass - - for script in scripts: - exe = "precise-{}".format(script.replace('_', '-')) - if filecontains(completed_file, exe): - continue - with tempfile.NamedTemporaryFile(mode = 'w',suffix='.spec', delete=False) as temp_file: - temp_path = temp_file.name - with open("precise.template.spec") as template_spec: - spec = template_spec.read() \ - .replace("%%SCRIPT%%",script) \ - .replace("%%TRAIN_LIBS%%", train_libs) \ - .replace("%%STRIP%%", "False") - temp_file.write(spec + '\n') - if os.system("pyinstaller -y {} --workpath=build/{}".format(temp_path, exe)) != 0: - raise Exception("pyinstaller error") - print(temp_path) - if exe != combined_folder: - dir_util.copy_tree(join("dist",exe), join("dist",combined_folder), update=1) - shutil.rmtree(join("dist",exe)) - shutil.rmtree(join("build",exe)) - with open(completed_file, 'a') as f: - f.write(exe + '\n') - - out_name = tar_name(tar_prefix) - make_tarfile(join('dist', out_name), join('dist', combined_folder)) - with open(join('dist', "{}.md5".format(out_name)), 'w') as md5file: - md5file.write(filemd5(join('dist', out_name))) +def delete_dir_if_exists(folder): + """Delete a folder, ignore if it does not exist. + + Args: + folder (str): The folder to be deleted. -def main(): - all_scripts=re.findall('(?<=precise.scripts.)[a-z_]+', open('setup.py').read()) - package_scripts("precise-all", "precise", all_scripts, "True") - package_scripts("precise-engine", "precise-engine", ["engine"], "False") + """ - tar_1 = join("dist",tar_name("precise-all")) - tar_2 = join("dist",tar_name("precise-engine")) - print("Wrote to {} and {}".format(tar_1, tar_2)) + try: + shutil.rmtree(folder) + except FileNotFoundError: + pass + +def createSpecFile(script, train_libs): + """Create a pyinstaller spec file based on precise.template.spec. + + Args: + script (str): the python script for which this spec file is intended. + train_libs (bool): whether the spec should include the training libs. + + Returns: + The path of the created spec file. + + """ + with tempfile.NamedTemporaryFile(mode = 'w',suffix='.spec', delete=False) as temp_file: + spec_path = temp_file.name + with open("precise.template.spec") as template_spec: + spec = template_spec.read() \ + .replace("%%SCRIPT%%",script) \ + .replace("%%TRAIN_LIBS%%", str(train_libs)) \ + .replace("%%STRIP%%", "False") + temp_file.write(spec + '\n') + return spec_path if __name__ == '__main__': main() From 88ae99319bc043409e2d999da4279adc7ec37ae0 Mon Sep 17 00:00:00 2001 From: NoamDev <37066741+NoamDev@users.noreply.github.com> Date: Tue, 22 Sep 2020 15:44:50 +0300 Subject: [PATCH 3/5] docstring typo --- win-package.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/win-package.py b/win-package.py index 029e0353..33cc4ab7 100644 --- a/win-package.py +++ b/win-package.py @@ -110,7 +110,7 @@ def filecontains(path, string): string (str): The phrase to look for. Returns: - True of the given file contains the given phrase, False otherwise. + True if the given file contains the given phrase, False otherwise. """ try: From 2b60e94820f0f3184f4206ae3c82820e0d5c0d6b Mon Sep 17 00:00:00 2001 From: NoamDev <37066741+NoamDev@users.noreply.github.com> Date: Tue, 22 Sep 2020 15:56:19 +0300 Subject: [PATCH 4/5] make sure batch scripts exit with 1 on error --- build.bat | 8 ++++---- setup.bat | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/build.bat b/build.bat index c624906e..5906cf43 100644 --- a/build.bat +++ b/build.bat @@ -1,5 +1,5 @@ @echo off -call setup.bat || exit /b -call .venv/Scripts/activate || exit /b -pip install pyinstaller || exit /b -python win-package.py || exit /b \ No newline at end of file +call setup.bat || exit /b 1 +call .venv/Scripts/activate || exit /b 1 +pip install pyinstaller || exit /b 1 +python win-package.py || exit /b 1 \ No newline at end of file diff --git a/setup.bat b/setup.bat index e75f6303..32ebee82 100644 --- a/setup.bat +++ b/setup.bat @@ -1,9 +1,9 @@ @echo off if not exist ".venv" ( - py -3.6 -m venv .venv || exit /b + py -3.6 -m venv .venv || exit /b 1 ) -call .venv/Scripts/activate || exit /b -pip install -e runner/ || exit /b +call .venv/Scripts/activate || exit /b 1 +pip install -e runner/ || exit /b 1 pip install -e . || exit /b rem Optional, for comparison -pip install pocketsphinx || exit /b \ No newline at end of file +pip install pocketsphinx || exit /b 1 \ No newline at end of file From 9c2eca94459a93dff24f36013d87b4923ee31c75 Mon Sep 17 00:00:00 2001 From: NoamDev <37066741+NoamDev@users.noreply.github.com> Date: Tue, 22 Sep 2020 16:00:53 +0300 Subject: [PATCH 5/5] fix arguments order --- win-package.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/win-package.py b/win-package.py index 33cc4ab7..99635b4f 100644 --- a/win-package.py +++ b/win-package.py @@ -54,7 +54,7 @@ def package_scripts(tar_prefix, combined_folder, scripts, train_libs): f.write(exe + '\n') out_name = tar_name(tar_prefix) - make_tarfile(join('dist', out_name), join('dist', combined_folder)) + make_tarfile(join('dist', combined_folder), join('dist', out_name)) with open(join('dist', "{}.md5".format(out_name)), 'w') as md5file: md5file.write(filemd5(join('dist', out_name)))