From 3a8ea868fef058cc8d9a27732baeb651e8587030 Mon Sep 17 00:00:00 2001 From: M Hightower <27247790+mhightower83@users.noreply.github.com> Date: Mon, 16 Jan 2023 17:09:40 -0800 Subject: [PATCH 1/4] Fix for Windows 11 preferences.txt not found - maybe Works on Windows 10. I don't have an 11 platform to test on. Uses the prefered method for finding path to `AppData\Local` Cleaned up return with undefined fqfn2. --- tools/mkbuildoptglobals.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/tools/mkbuildoptglobals.py b/tools/mkbuildoptglobals.py index 5c98ce3f7a..38877e1d7d 100644 --- a/tools/mkbuildoptglobals.py +++ b/tools/mkbuildoptglobals.py @@ -447,23 +447,22 @@ def find_preferences_txt(runtime_ide_path): # verified on Windows 10 with Arduino IDE 1.8.19 if os.path.exists(fqfn): return [fqfn, None] + # It is never simple. Arduino from the Windows APP store or the download # Windows 8 and up option will save "preferences.txt" in one location. # The downloaded Windows 7 (and up version) will put "preferences.txt" # in a different location. When both are present due to various possible - # scenarios, use the more modern. + # scenarios, give preference to the APP store. fqfn = os.path.expanduser("~\Documents\ArduinoData\preferences.txt") # Path for "Windows app" - verified on Windows 10 with Arduino IDE 1.8.19 from APP store - fqfn2 = os.path.expanduser("~\AppData\local\Arduino15\preferences.txt") + fqfn2 = os.path.normpath(os.getenv("LOCALAPPDATA") + "\Arduino15\preferences.txt") # Path for Windows 7 and up - verified on Windows 10 with Arduino IDE 1.8.19 if os.path.exists(fqfn): if os.path.exists(fqfn2): print_err("Multiple 'preferences.txt' files found:") print_err(" " + fqfn) print_err(" " + fqfn2) - return [fqfn, None] - else: - return [fqfn, fqfn2] + return [fqfn, None] elif os.path.exists(fqfn2): return [fqfn2, None] elif "Darwin" == platform_name: From c7fb78a30f4947144f166b7d511fa1f56863f942 Mon Sep 17 00:00:00 2001 From: M Hightower <27247790+mhightower83@users.noreply.github.com> Date: Tue, 17 Jan 2023 09:02:02 -0800 Subject: [PATCH 2/4] Correct the correction --- tools/mkbuildoptglobals.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tools/mkbuildoptglobals.py b/tools/mkbuildoptglobals.py index 38877e1d7d..b70ba08e7a 100644 --- a/tools/mkbuildoptglobals.py +++ b/tools/mkbuildoptglobals.py @@ -447,7 +447,6 @@ def find_preferences_txt(runtime_ide_path): # verified on Windows 10 with Arduino IDE 1.8.19 if os.path.exists(fqfn): return [fqfn, None] - # It is never simple. Arduino from the Windows APP store or the download # Windows 8 and up option will save "preferences.txt" in one location. # The downloaded Windows 7 (and up version) will put "preferences.txt" @@ -462,7 +461,9 @@ def find_preferences_txt(runtime_ide_path): print_err("Multiple 'preferences.txt' files found:") print_err(" " + fqfn) print_err(" " + fqfn2) - return [fqfn, None] + return [fqfn, fqfn2] + else: + return [fqfn, None] elif os.path.exists(fqfn2): return [fqfn2, None] elif "Darwin" == platform_name: From 956dd59f3bd28c472fd2d25bce7c19d52eca329f Mon Sep 17 00:00:00 2001 From: M Hightower <27247790+mhightower83@users.noreply.github.com> Date: Tue, 17 Jan 2023 13:57:50 -0800 Subject: [PATCH 3/4] New handling of missing perferences.txt Draft: alternative option for handling missing perferences.txt - safe-mode --- tools/mkbuildoptglobals.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/tools/mkbuildoptglobals.py b/tools/mkbuildoptglobals.py index b70ba08e7a..3b6ae91f36 100644 --- a/tools/mkbuildoptglobals.py +++ b/tools/mkbuildoptglobals.py @@ -510,15 +510,17 @@ def check_preferences_txt(runtime_ide_path, preferences_file): # This should never fail. file_fqfn = find_preferences_txt(runtime_ide_path) if file_fqfn[0] != None: - print_msg(f"Using preferences from '{file_fqfn[0]}'") val0 = get_preferences_txt(file_fqfn[0], key) val1 = val0 if file_fqfn[1] != None: val1 = get_preferences_txt(file_fqfn[1], key) if val0 == val1: # We can safely ignore that there were two preferences.txt files + print_msg(f"Using preferences from '{file_fqfn[0]}'") return val0 else: print_err(f"Found too many preferences.txt files with different values for '{key}'") + print_err(f" '{file_fqfn[0]}'") + print_err(f" '{file_fqfn[1]}'") raise UserWarning else: # Something is wrong with the installation or our understanding of the installation. @@ -595,13 +597,13 @@ def determine_cache_state(args, runtime_ide_path, source_globals_h_fqfn): caching_enabled = check_preferences_txt(ide_path, preferences_fqfn) except UserWarning: if os.path.exists(source_globals_h_fqfn): - caching_enabled = None print_err(f" runtime_ide_version: {args.runtime_ide_version}") - print_err(f" This must be resolved to use '{globals_name}'") + print_err(f" When using '{globals_name}', resolve these issues for better build performance.") + print_err(f" Using safe-mode") print_err(f" Read more at {docs_url}") - else: - # We can quietly ignore the problem because we are not needed. - caching_enabled = True + # fall-through fail case - assume the worst case - safe-mode + # else case - We can quietly ignore the problem because we are not needed + caching_enabled = True return caching_enabled @@ -742,7 +744,7 @@ def main(): use_aggressive_caching_workaround = determine_cache_state(args, runtime_ide_path, source_globals_h_fqfn) if use_aggressive_caching_workaround == None: - # Specific rrror messages already buffered + # Specific error messages already buffered handle_error(1) print_dbg(f"first_time: {first_time}") From 998c071f4a95ef2ae14b3c9649e7e0e9f69be634 Mon Sep 17 00:00:00 2001 From: M Hightower <27247790+mhightower83@users.noreply.github.com> Date: Tue, 17 Jan 2023 21:46:53 -0800 Subject: [PATCH 4/4] Using a file descriptor argument with os.utime on Windows OS is unsupported. Updated usage. Removed using IDE version as an indicator of shared "core.a" caching. Assume shared "core.a" caching unless directed otherwise. --- tools/mkbuildoptglobals.py | 128 +++++-------------------------------- 1 file changed, 16 insertions(+), 112 deletions(-) diff --git a/tools/mkbuildoptglobals.py b/tools/mkbuildoptglobals.py index 3b6ae91f36..afbbfb833c 100644 --- a/tools/mkbuildoptglobals.py +++ b/tools/mkbuildoptglobals.py @@ -415,69 +415,6 @@ def discover_1st_time_run(build_path): count += 1 return 0 == count - -def find_preferences_txt(runtime_ide_path): - """ - Check for perferences.txt in well-known locations. Most OSs have two - possibilities. When "portable" is present, it takes priority. Otherwise, the - remaining path wins. However, Windows has two. Depending on the install - source, the APP store or website download, both may appear and create an - ambiguous result. - - Return two item list - Two non "None" items indicate an ambiguous state. - - OS Path list for Arduino IDE 1.6.0 and newer - from: https://www.arduino.cc/en/hacking/preferences - """ - platform_name = platform.system() - if "Linux" == platform_name: - # Test for portable 1ST - # /portable/preferences.txt (when used in portable mode) - # For more on portable mode see https://docs.arduino.cc/software/ide-v1/tutorials/PortableIDE - fqfn = os.path.normpath(runtime_ide_path + "/portable/preferences.txt") - # Linux - verified with Arduino IDE 1.8.19 - if os.path.exists(fqfn): - return [fqfn, None] - fqfn = os.path.expanduser("~/.arduino15/preferences.txt") - # Linux - verified with Arduino IDE 1.8.18 and 2.0 RC5 64bit and AppImage - if os.path.exists(fqfn): - return [fqfn, None] - elif "Windows" == platform_name: - fqfn = os.path.normpath(runtime_ide_path + "\portable\preferences.txt") - # verified on Windows 10 with Arduino IDE 1.8.19 - if os.path.exists(fqfn): - return [fqfn, None] - # It is never simple. Arduino from the Windows APP store or the download - # Windows 8 and up option will save "preferences.txt" in one location. - # The downloaded Windows 7 (and up version) will put "preferences.txt" - # in a different location. When both are present due to various possible - # scenarios, give preference to the APP store. - fqfn = os.path.expanduser("~\Documents\ArduinoData\preferences.txt") - # Path for "Windows app" - verified on Windows 10 with Arduino IDE 1.8.19 from APP store - fqfn2 = os.path.normpath(os.getenv("LOCALAPPDATA") + "\Arduino15\preferences.txt") - # Path for Windows 7 and up - verified on Windows 10 with Arduino IDE 1.8.19 - if os.path.exists(fqfn): - if os.path.exists(fqfn2): - print_err("Multiple 'preferences.txt' files found:") - print_err(" " + fqfn) - print_err(" " + fqfn2) - return [fqfn, fqfn2] - else: - return [fqfn, None] - elif os.path.exists(fqfn2): - return [fqfn2, None] - elif "Darwin" == platform_name: - # Portable is not compatable with Mac OS X - # see https://docs.arduino.cc/software/ide-v1/tutorials/PortableIDE - fqfn = os.path.expanduser("~/Library/Arduino15/preferences.txt") - # Mac OS X - unverified - if os.path.exists(fqfn): - return [fqfn, None] - - print_err("File preferences.txt not found on " + platform_name) - return [None, None] - - def get_preferences_txt(file_fqfn, key): # Get Key Value, key is allowed to be missing. # We assume file file_fqfn exists @@ -505,42 +442,28 @@ def check_preferences_txt(runtime_ide_path, preferences_file): else: print_err(f"Override preferences file '{preferences_file}' not found.") - elif runtime_ide_path != None: - # For a particular install, search the expected locations for platform.txt - # This should never fail. - file_fqfn = find_preferences_txt(runtime_ide_path) - if file_fqfn[0] != None: - val0 = get_preferences_txt(file_fqfn[0], key) - val1 = val0 - if file_fqfn[1] != None: - val1 = get_preferences_txt(file_fqfn[1], key) - if val0 == val1: # We can safely ignore that there were two preferences.txt files - print_msg(f"Using preferences from '{file_fqfn[0]}'") - return val0 - else: - print_err(f"Found too many preferences.txt files with different values for '{key}'") - print_err(f" '{file_fqfn[0]}'") - print_err(f" '{file_fqfn[1]}'") - raise UserWarning - else: - # Something is wrong with the installation or our understanding of the installation. - print_err("'preferences.txt' file missing from well known locations.") - - return None - + # Referencing the preferences.txt for an indication of shared "core.a" + # caching is unreliable. There are too many places reference.txt can be + # stored and no hints of which the Arduino build might be using. Unless + # directed otherwise, assume "core.a" caching true. + print_msg(f"Assume aggressive 'core.a' caching enabled.") + return True def touch(fname, times=None): with open(fname, "ab") as file: - os.utime(file.fileno(), times) + file.close(); + os.utime(fname, times) def synchronous_touch(globals_h_fqfn, commonhfile_fqfn): global debug_enabled # touch both files with the same timestamp touch(globals_h_fqfn) with open(globals_h_fqfn, "rb") as file: - ts = os.stat(file.fileno()) - with open(commonhfile_fqfn, "ab") as file2: - os.utime(file2.fileno(), ns=(ts.st_atime_ns, ts.st_mtime_ns)) + file.close() + with open(commonhfile_fqfn, "ab") as file2: + file2.close() + ts = os.stat(globals_h_fqfn) + os.utime(commonhfile_fqfn, ns=(ts.st_atime_ns, ts.st_mtime_ns)) if debug_enabled: print_dbg("After synchronous_touch") @@ -554,12 +477,8 @@ def synchronous_touch(globals_h_fqfn, commonhfile_fqfn): def determine_cache_state(args, runtime_ide_path, source_globals_h_fqfn): global docs_url print_dbg(f"runtime_ide_version: {args.runtime_ide_version}") - if args.runtime_ide_version < 10802: # CI also has version 10607 -- and args.runtime_ide_version != 10607: - # Aggresive core caching - not implemented before version 1.8.2 - # Note, Arduino IDE 2.0 rc5 has version 1.6.7 and has aggressive caching. - print_dbg(f"Old version ({args.runtime_ide_version}) of Arduino IDE no aggressive caching option") - return False - elif args.cache_core != None: + + if args.cache_core != None: print_msg(f"Preferences override, this prebuild script assumes the 'compiler.cache_core' parameter is set to {args.cache_core}") print_msg(f"To change, modify 'mkbuildoptglobals.extra_flags=(--cache_core | --no_cache_core)' in 'platform.local.txt'") return args.cache_core @@ -593,19 +512,7 @@ def determine_cache_state(args, runtime_ide_path, source_globals_h_fqfn): preferences_fqfn = os.path.expanduser(preferences_fqfn) print_dbg(f"determine_cache_state: preferences_fqfn: {preferences_fqfn}") - try: - caching_enabled = check_preferences_txt(ide_path, preferences_fqfn) - except UserWarning: - if os.path.exists(source_globals_h_fqfn): - print_err(f" runtime_ide_version: {args.runtime_ide_version}") - print_err(f" When using '{globals_name}', resolve these issues for better build performance.") - print_err(f" Using safe-mode") - print_err(f" Read more at {docs_url}") - # fall-through fail case - assume the worst case - safe-mode - # else case - We can quietly ignore the problem because we are not needed - caching_enabled = True - - return caching_enabled + return check_preferences_txt(ide_path, preferences_fqfn) """ @@ -743,9 +650,6 @@ def main(): print_dbg("First run since Arduino IDE started.") use_aggressive_caching_workaround = determine_cache_state(args, runtime_ide_path, source_globals_h_fqfn) - if use_aggressive_caching_workaround == None: - # Specific error messages already buffered - handle_error(1) print_dbg(f"first_time: {first_time}") print_dbg(f"use_aggressive_caching_workaround: {use_aggressive_caching_workaround}")