Merge pull request #12 from kryksyh/bugfixes

Bugfixes

.github/workflows/cmake.yml

@@ -6,10 +6,8 @@ on:
   workflow_dispatch:
 
 env:
-  # Customize the CMake build type here (Release, Debug, RelWithDebInfo, etc.)
   BUILD_TYPE: Release
-  CONAN_USER_HOME: ${{ github.workspace }}/conan-home/
-  CONAN_USER_HOME_SHORT: "${{ github.workspace }}/conan-home/short"
+
 jobs:
   build:
     name: Building ${{ matrix.config.os }}
@@ -18,58 +16,51 @@ jobs:
       matrix:
         config:
         - os: ubuntu-latest
-          generator: Unix Makefiles
-        #- os: macos-11
-        #  generator: Xcode
+          preset: conan-release
+        - os: macos-latest
+          preset: conan-release
         - os: windows-2022
-          generator: Visual Studio 17 2022
+          preset: conan-default
     steps:
-    - name: switch to gcc-11 on linux
-      if: ${{ matrix.config.os == 'ubuntu-latest' }}
-      run: |
-        sudo apt install gcc-11 g++-11
-        sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-11 100 --slave /usr/bin/g++ g++ /usr/bin/g++-11 --slave /usr/bin/gcov gcov /usr/bin/gcov-11
-        sudo update-alternatives --set gcc /usr/bin/gcc-11
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
+
     - name: Setup Python
-      uses: actions/setup-python@v2.3.1
+      uses: actions/setup-python@v5
       with:
-        cache: pip
-        #cache-dependency-path: # optional
-    - uses: BSFishy/pip-action@v1
+        python-version: '3.12'
+
+    - name: Install Conan
+      run: pip install "conan>=2"
+
+    - name: Detect Conan profile
+      run: conan profile detect --force
+
+    - name: Cache Conan packages
+      uses: actions/cache@v4
       with:
-        packages: conan
-    - name: Cache for .conan
-      id: cache-conan
-      uses: actions/cache@v2
-      env:
-        cache-name: cache-conan-modules
-      with:
-        path: ${{ env.CONAN_USER_HOME }}
-        key: host-${{ matrix.config.os }}-${{ hashFiles('requirements.txt') }}-${{ hashFiles('CMakeLists.txt') }}
+        path: ~/.conan2
+        key: conan-${{ matrix.config.os }}-${{ hashFiles('conanfile.txt') }}
         restore-keys: |
-          host-3-${{ matrix.config.os }}-
+          conan-${{ matrix.config.os }}-
+
+    - name: Install dependencies
+      run: conan install . --output-folder=build/${{ env.BUILD_TYPE }} --build=missing -s build_type=${{ env.BUILD_TYPE }}
 
     - name: Configure CMake
-      # Configure CMake in a 'build' subdirectory. `CMAKE_BUILD_TYPE` is only required if you are using a single-configuration generator such as make.
-      # See https://cmake.org/cmake/help/latest/variable/CMAKE_BUILD_TYPE.html?highlight=cmake_build_type
-      run: cmake -B ${{github.workspace}}/build -G"${{ matrix.config.generator }}" -DCMAKE_BUILD_TYPE=${{ env.BUILD_TYPE }} -DCMAKE_CONFIGURATION_TYPES=${{ env.BUILD_TYPE }}
+      run: cmake --preset ${{ matrix.config.preset }}
 
     - name: Build
-      # Build your program with the given configuration
-      run: cmake --build ${{github.workspace}}/build --config ${{env.BUILD_TYPE}}
+      run: cmake --build build/${{ env.BUILD_TYPE }} --config ${{ env.BUILD_TYPE }}
 
     - name: Package
-      # Build your program with the given configuration
-      run: |
-        cd build
-        cmake --build . --target package --config "${{ env.BUILD_TYPE }}"
+      working-directory: build/${{ env.BUILD_TYPE }}
+      run: cmake --build . --target package --config ${{ env.BUILD_TYPE }}
+
     - name: Upload artifact
-      uses: actions/upload-artifact@v2
+      uses: actions/upload-artifact@v4
       with:
         name: audacity-project-tools-${{ matrix.config.os }}
         path: |
-          build/packages/*
-          !build/packages/_CPack_Packages
+          build/${{ env.BUILD_TYPE }}/packages/*
+          !build/${{ env.BUILD_TYPE }}/packages/_CPack_Packages
         if-no-files-found: error
-

.github/workflows/release.yml

@@ -0,0 +1,53 @@
+name: Release
+
+on:
+  workflow_dispatch:
+    inputs:
+      run_id:
+        description: 'Workflow run ID to take artifacts from'
+        required: true
+        type: string
+      tag:
+        description: 'Tag name (e.g. v1.0.5)'
+        required: true
+        type: string
+      prerelease:
+        description: 'Mark as pre-release'
+        required: false
+        type: boolean
+        default: false
+
+jobs:
+  release:
+    name: Create Release
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+    steps:
+    - uses: actions/checkout@v4
+
+    - name: Download artifacts
+      env:
+        GH_TOKEN: ${{ github.token }}
+      run: |
+        mkdir -p release-assets
+        gh run download ${{ inputs.run_id }} --dir release-artifacts
+
+        for dir in release-artifacts/*/; do
+          name=$(basename "$dir")
+          (cd "$dir" && zip -r "../../release-assets/${name}.zip" .)
+        done
+
+    - name: Create tag
+      run: |
+        git tag ${{ inputs.tag }}
+        git push origin ${{ inputs.tag }}
+
+    - name: Create release
+      env:
+        GH_TOKEN: ${{ github.token }}
+      run: |
+        gh release create ${{ inputs.tag }} \
+          --title "${{ inputs.tag }}" \
+          ${{ inputs.prerelease && '--prerelease' || '' }} \
+          release-assets/*

.gitignore

@@ -34,3 +34,14 @@
 build/**
 .vs/**
 out/**
+.venv/**
+CMakeUserPresets.json
+
+# Audacity files
+
+*.aup3
+*.aup4
+*.aup3-shm
+*.aup3-wal
+*.aup4-shm
+*.aup4-wal

3party/sqlite3/sqlite3.h

@@ -146,9 +146,9 @@ extern "C" {
 ** [sqlite3_libversion_number()], [sqlite3_sourceid()],
 ** [sqlite_version()] and [sqlite_source_id()].
 */
-#define SQLITE_VERSION        "3.37.0"
-#define SQLITE_VERSION_NUMBER 3037000
-#define SQLITE_SOURCE_ID      "2021-11-27 14:13:22 bd41822c7424d393a30e92ff6cb254d25c26769889c1499a18a0b9339f5d6c8a"
+#define SQLITE_VERSION        "3.40.1"
+#define SQLITE_VERSION_NUMBER 3040001
+#define SQLITE_SOURCE_ID      "2022-12-28 14:03:47 df5c253c0b3dd24916e4ec7cf77d3db5294cc9fd45ae7b9c5e82ad8197f38a24"
 
 /*
 ** CAPI3REF: Run-Time Library Version Numbers
@@ -566,7 +566,7 @@ SQLITE_API int sqlite3_exec(
 #define SQLITE_WARNING_AUTOINDEX       (SQLITE_WARNING | (1<<8))
 #define SQLITE_AUTH_USER               (SQLITE_AUTH | (1<<8))
 #define SQLITE_OK_LOAD_PERMANENTLY     (SQLITE_OK | (1<<8))
-#define SQLITE_OK_SYMLINK              (SQLITE_OK | (2<<8))
+#define SQLITE_OK_SYMLINK              (SQLITE_OK | (2<<8)) /* internal use only */
 
 /*
 ** CAPI3REF: Flags For File Open Operations
@@ -670,13 +670,17 @@ SQLITE_API int sqlite3_exec(
 **
 ** SQLite uses one of these integer values as the second
 ** argument to calls it makes to the xLock() and xUnlock() methods
-** of an [sqlite3_io_methods] object.
+** of an [sqlite3_io_methods] object.  These values are ordered from
+** lest restrictive to most restrictive.
+**
+** The argument to xLock() is always SHARED or higher.  The argument to
+** xUnlock is either SHARED or NONE.
 */
-#define SQLITE_LOCK_NONE          0
-#define SQLITE_LOCK_SHARED        1
-#define SQLITE_LOCK_RESERVED      2
-#define SQLITE_LOCK_PENDING       3
-#define SQLITE_LOCK_EXCLUSIVE     4
+#define SQLITE_LOCK_NONE          0       /* xUnlock() only */
+#define SQLITE_LOCK_SHARED        1       /* xLock() or xUnlock() */
+#define SQLITE_LOCK_RESERVED      2       /* xLock() only */
+#define SQLITE_LOCK_PENDING       3       /* xLock() only */
+#define SQLITE_LOCK_EXCLUSIVE     4       /* xLock() only */
 
 /*
 ** CAPI3REF: Synchronization Type Flags
@@ -754,7 +758,14 @@ struct sqlite3_file {
 ** <li> [SQLITE_LOCK_PENDING], or
 ** <li> [SQLITE_LOCK_EXCLUSIVE].
 ** </ul>
-** xLock() increases the lock. xUnlock() decreases the lock.
+** xLock() upgrades the database file lock.  In other words, xLock() moves the
+** database file lock in the direction NONE toward EXCLUSIVE. The argument to
+** xLock() is always on of SHARED, RESERVED, PENDING, or EXCLUSIVE, never
+** SQLITE_LOCK_NONE.  If the database file lock is already at or above the
+** requested lock, then the call to xLock() is a no-op.
+** xUnlock() downgrades the database file lock to either SHARED or NONE.
+*  If the lock is already at or below the requested lock state, then the call
+** to xUnlock() is a no-op.
 ** The xCheckReservedLock() method checks whether any database connection,
 ** either in this process or in some other process, is holding a RESERVED,
 ** PENDING, or EXCLUSIVE lock on the file.  It returns true
@@ -859,9 +870,8 @@ struct sqlite3_io_methods {
 ** opcode causes the xFileControl method to write the current state of
 ** the lock (one of [SQLITE_LOCK_NONE], [SQLITE_LOCK_SHARED],
 ** [SQLITE_LOCK_RESERVED], [SQLITE_LOCK_PENDING], or [SQLITE_LOCK_EXCLUSIVE])
-** into an integer that the pArg argument points to. This capability
-** is used during testing and is only available when the SQLITE_TEST
-** compile-time option is used.
+** into an integer that the pArg argument points to.
+** This capability is only available if SQLite is compiled with [SQLITE_DEBUG].
 **
 ** <li>[[SQLITE_FCNTL_SIZE_HINT]]
 ** The [SQLITE_FCNTL_SIZE_HINT] opcode is used by SQLite to give the VFS
@@ -1182,6 +1192,12 @@ struct sqlite3_io_methods {
 **
 ** <li>[[SQLITE_FCNTL_CKSM_FILE]]
 ** Used by the cksmvfs VFS module only.
+**
+** <li>[[SQLITE_FCNTL_RESET_CACHE]]
+** If there is currently no transaction open on the database, and the
+** database is not a temp db, then this file-control purges the contents
+** of the in-memory page cache. If there is an open transaction, or if
+** the db is a temp-db, it is a no-op, not an error.
 ** </ul>
 */
 #define SQLITE_FCNTL_LOCKSTATE               1
@@ -1224,6 +1240,7 @@ struct sqlite3_io_methods {
 #define SQLITE_FCNTL_CKPT_START             39
 #define SQLITE_FCNTL_EXTERNAL_READER        40
 #define SQLITE_FCNTL_CKSM_FILE              41
+#define SQLITE_FCNTL_RESET_CACHE            42
 
 /* deprecated names */
 #define SQLITE_GET_LOCKPROXYFILE      SQLITE_FCNTL_GET_LOCKPROXYFILE
@@ -1253,6 +1270,26 @@ typedef struct sqlite3_mutex sqlite3_mutex;
 */
 typedef struct sqlite3_api_routines sqlite3_api_routines;
 
+/*
+** CAPI3REF: File Name
+**
+** Type [sqlite3_filename] is used by SQLite to pass filenames to the
+** xOpen method of a [VFS]. It may be cast to (const char*) and treated
+** as a normal, nul-terminated, UTF-8 buffer containing the filename, but
+** may also be passed to special APIs such as:
+**
+** <ul>
+** <li>  sqlite3_filename_database()
+** <li>  sqlite3_filename_journal()
+** <li>  sqlite3_filename_wal()
+** <li>  sqlite3_uri_parameter()
+** <li>  sqlite3_uri_boolean()
+** <li>  sqlite3_uri_int64()
+** <li>  sqlite3_uri_key()
+** </ul>
+*/
+typedef const char *sqlite3_filename;
+
 /*
 ** CAPI3REF: OS Interface Object
 **
@@ -1431,7 +1468,7 @@ struct sqlite3_vfs {
   sqlite3_vfs *pNext;      /* Next registered VFS */
   const char *zName;       /* Name of this virtual file system */
   void *pAppData;          /* Pointer to application-specific data */
-  int (*xOpen)(sqlite3_vfs*, const char *zName, sqlite3_file*,
+  int (*xOpen)(sqlite3_vfs*, sqlite3_filename zName, sqlite3_file*,
                int flags, int *pOutFlags);
   int (*xDelete)(sqlite3_vfs*, const char *zName, int syncDir);
   int (*xAccess)(sqlite3_vfs*, const char *zName, int flags, int *pResOut);
@@ -2309,6 +2346,7 @@ struct sqlite3_mem_methods {
 ** <ul>
 ** <li> The [PRAGMA writable_schema=ON] statement.
 ** <li> The [PRAGMA journal_mode=OFF] statement.
+** <li> The [PRAGMA schema_version=N] statement.
 ** <li> Writes to the [sqlite_dbpage] virtual table.
 ** <li> Direct writes to [shadow tables].
 ** </ul>
@@ -3424,6 +3462,9 @@ SQLITE_API void sqlite3_progress_handler(sqlite3*, int, int(*)(void*), void*);
 ** <dd>The database is opened [shared cache] enabled, overriding
 ** the default shared cache setting provided by
 ** [sqlite3_enable_shared_cache()].)^
+** The [use of shared cache mode is discouraged] and hence shared cache
+** capabilities may be omitted from many builds of SQLite.  In such cases,
+** this option is a no-op.
 **
 ** ^(<dt>[SQLITE_OPEN_PRIVATECACHE]</dt>
 ** <dd>The database is opened [shared cache] disabled, overriding
@@ -3439,7 +3480,7 @@ SQLITE_API void sqlite3_progress_handler(sqlite3*, int, int(*)(void*), void*);
 ** to return an extended result code.</dd>
 **
 ** [[OPEN_NOFOLLOW]] ^(<dt>[SQLITE_OPEN_NOFOLLOW]</dt>
-** <dd>The database filename is not allowed to be a symbolic link</dd>
+** <dd>The database filename is not allowed to contain a symbolic link</dd>
 ** </dl>)^
 **
 ** If the 3rd parameter to sqlite3_open_v2() is not one of the
@@ -3698,10 +3739,10 @@ SQLITE_API int sqlite3_open_v2(
 **
 ** See the [URI filename] documentation for additional information.
 */
-SQLITE_API const char *sqlite3_uri_parameter(const char *zFilename, const char *zParam);
-SQLITE_API int sqlite3_uri_boolean(const char *zFile, const char *zParam, int bDefault);
-SQLITE_API sqlite3_int64 sqlite3_uri_int64(const char*, const char*, sqlite3_int64);
-SQLITE_API const char *sqlite3_uri_key(const char *zFilename, int N);
+SQLITE_API const char *sqlite3_uri_parameter(sqlite3_filename z, const char *zParam);
+SQLITE_API int sqlite3_uri_boolean(sqlite3_filename z, const char *zParam, int bDefault);
+SQLITE_API sqlite3_int64 sqlite3_uri_int64(sqlite3_filename, const char*, sqlite3_int64);
+SQLITE_API const char *sqlite3_uri_key(sqlite3_filename z, int N);
 
 /*
 ** CAPI3REF:  Translate filenames
@@ -3730,9 +3771,9 @@ SQLITE_API const char *sqlite3_uri_key(const char *zFilename, int N);
 ** return value from [sqlite3_db_filename()], then the result is
 ** undefined and is likely a memory access violation.
 */
-SQLITE_API const char *sqlite3_filename_database(const char*);
-SQLITE_API const char *sqlite3_filename_journal(const char*);
-SQLITE_API const char *sqlite3_filename_wal(const char*);
+SQLITE_API const char *sqlite3_filename_database(sqlite3_filename);
+SQLITE_API const char *sqlite3_filename_journal(sqlite3_filename);
+SQLITE_API const char *sqlite3_filename_wal(sqlite3_filename);
 
 /*
 ** CAPI3REF:  Database File Corresponding To A Journal
@@ -3798,14 +3839,14 @@ SQLITE_API sqlite3_file *sqlite3_database_file_object(const char*);
 ** then the corresponding [sqlite3_module.xClose() method should also be
 ** invoked prior to calling sqlite3_free_filename(Y).
 */
-SQLITE_API char *sqlite3_create_filename(
+SQLITE_API sqlite3_filename sqlite3_create_filename(
   const char *zDatabase,
   const char *zJournal,
   const char *zWal,
   int nParam,
   const char **azParam
 );
-SQLITE_API void sqlite3_free_filename(char*);
+SQLITE_API void sqlite3_free_filename(sqlite3_filename);
 
 /*
 ** CAPI3REF: Error Codes And Messages
@@ -3824,13 +3865,14 @@ SQLITE_API void sqlite3_free_filename(char*);
 ** sqlite3_extended_errcode() might change with each API call.
 ** Except, there are some interfaces that are guaranteed to never
 ** change the value of the error code.  The error-code preserving
-** interfaces are:
+** interfaces include the following:
 **
 ** <ul>
 ** <li> sqlite3_errcode()
 ** <li> sqlite3_extended_errcode()
 ** <li> sqlite3_errmsg()
 ** <li> sqlite3_errmsg16()
+** <li> sqlite3_error_offset()
 ** </ul>
 **
 ** ^The sqlite3_errmsg() and sqlite3_errmsg16() return English-language
@@ -3845,6 +3887,13 @@ SQLITE_API void sqlite3_free_filename(char*);
 ** ^(Memory to hold the error message string is managed internally
 ** and must not be freed by the application)^.
 **
+** ^If the most recent error references a specific token in the input
+** SQL, the sqlite3_error_offset() interface returns the byte offset
+** of the start of that token.  ^The byte offset returned by
+** sqlite3_error_offset() assumes that the input SQL is UTF8.
+** ^If the most recent error does not reference a specific token in the input
+** SQL, then the sqlite3_error_offset() function returns -1.
+**
 ** When the serialized [threading mode] is in use, it might be the
 ** case that a second error occurs on a separate thread in between
 ** the time of the first error and the call to these interfaces.
@@ -3864,6 +3913,7 @@ SQLITE_API int sqlite3_extended_errcode(sqlite3 *db);
 SQLITE_API const char *sqlite3_errmsg(sqlite3*);
 SQLITE_API const void *sqlite3_errmsg16(sqlite3*);
 SQLITE_API const char *sqlite3_errstr(int);
+SQLITE_API int sqlite3_error_offset(sqlite3 *db);
 
 /*
 ** CAPI3REF: Prepared Statement Object
@@ -4275,6 +4325,10 @@ SQLITE_API const char *sqlite3_normalized_sql(sqlite3_stmt *pStmt);
 ** be false.  ^Similarly, a CREATE TABLE IF NOT EXISTS statement is a
 ** read-only no-op if the table already exists, but
 ** sqlite3_stmt_readonly() still returns false for such a statement.
+**
+** ^If prepared statement X is an [EXPLAIN] or [EXPLAIN QUERY PLAN]
+** statement, then sqlite3_stmt_readonly(X) returns the same value as
+** if the EXPLAIN or EXPLAIN QUERY PLAN prefix were omitted.
 */
 SQLITE_API int sqlite3_stmt_readonly(sqlite3_stmt *pStmt);
 
@@ -4343,6 +4397,8 @@ SQLITE_API int sqlite3_stmt_busy(sqlite3_stmt*);
 **
 ** ^The sqlite3_value objects that are passed as parameters into the
 ** implementation of [application-defined SQL functions] are protected.
+** ^The sqlite3_value objects returned by [sqlite3_vtab_rhs_value()]
+** are protected.
 ** ^The sqlite3_value object returned by
 ** [sqlite3_column_value()] is unprotected.
 ** Unprotected sqlite3_value objects may only be used as arguments
@@ -4964,6 +5020,10 @@ SQLITE_API int sqlite3_data_count(sqlite3_stmt *pStmt);
 ** even empty strings, are always zero-terminated.  ^The return
 ** value from sqlite3_column_blob() for a zero-length BLOB is a NULL pointer.
 **
+** ^Strings returned by sqlite3_column_text16() always have the endianness
+** which is native to the platform, regardless of the text encoding set
+** for the database.
+**
 ** <b>Warning:</b> ^The object returned by [sqlite3_column_value()] is an
 ** [unprotected sqlite3_value] object.  In a multithreaded environment,
 ** an unprotected sqlite3_value object may only be used safely with
@@ -4977,7 +5037,7 @@ SQLITE_API int sqlite3_data_count(sqlite3_stmt *pStmt);
 ** [application-defined SQL functions] or [virtual tables], not within
 ** top-level application code.
 **
-** The these routines may attempt to convert the datatype of the result.
+** These routines may attempt to convert the datatype of the result.
 ** ^For example, if the internal representation is FLOAT and a text result
 ** is requested, [sqlite3_snprintf()] is used internally to perform the
 ** conversion automatically.  ^(The following table details the conversions
@@ -5002,7 +5062,7 @@ SQLITE_API int sqlite3_data_count(sqlite3_stmt *pStmt);
 ** <tr><td>  TEXT    <td>   BLOB    <td> No change
 ** <tr><td>  BLOB    <td> INTEGER   <td> [CAST] to INTEGER
 ** <tr><td>  BLOB    <td>  FLOAT    <td> [CAST] to REAL
-** <tr><td>  BLOB    <td>   TEXT    <td> Add a zero terminator if needed
+** <tr><td>  BLOB    <td>   TEXT    <td> [CAST] to TEXT, ensure zero terminator
 ** </table>
 ** </blockquote>)^
 **
@@ -5489,6 +5549,16 @@ SQLITE_API SQLITE_DEPRECATED int sqlite3_memory_alarm(void(*)(void*,sqlite3_int6
 ** then the conversion is performed.  Otherwise no conversion occurs.
 ** The [SQLITE_INTEGER | datatype] after conversion is returned.)^
 **
+** ^(The sqlite3_value_encoding(X) interface returns one of [SQLITE_UTF8],
+** [SQLITE_UTF16BE], or [SQLITE_UTF16LE] according to the current encoding
+** of the value X, assuming that X has type TEXT.)^  If sqlite3_value_type(X)
+** returns something other than SQLITE_TEXT, then the return value from
+** sqlite3_value_encoding(X) is meaningless.  ^Calls to
+** sqlite3_value_text(X), sqlite3_value_text16(X), sqlite3_value_text16be(X),
+** sqlite3_value_text16le(X), sqlite3_value_bytes(X), or
+** sqlite3_value_bytes16(X) might change the encoding of the value X and
+** thus change the return from subsequent calls to sqlite3_value_encoding(X).
+**
 ** ^Within the [xUpdate] method of a [virtual table], the
 ** sqlite3_value_nochange(X) interface returns true if and only if
 ** the column corresponding to X is unchanged by the UPDATE operation
@@ -5553,6 +5623,7 @@ SQLITE_API int sqlite3_value_type(sqlite3_value*);
 SQLITE_API int sqlite3_value_numeric_type(sqlite3_value*);
 SQLITE_API int sqlite3_value_nochange(sqlite3_value*);
 SQLITE_API int sqlite3_value_frombind(sqlite3_value*);
+SQLITE_API int sqlite3_value_encoding(sqlite3_value*);
 
 /*
 ** CAPI3REF: Finding The Subtype Of SQL Values
@@ -5574,7 +5645,8 @@ SQLITE_API unsigned int sqlite3_value_subtype(sqlite3_value*);
 ** object D and returns a pointer to that copy.  ^The [sqlite3_value] returned
 ** is a [protected sqlite3_value] object even if the input is not.
 ** ^The sqlite3_value_dup(V) interface returns NULL if V is NULL or if a
-** memory allocation fails.
+** memory allocation fails. ^If V is a [pointer value], then the result
+** of sqlite3_value_dup(V) is a NULL value.
 **
 ** ^The sqlite3_value_free(V) interface frees an [sqlite3_value] object
 ** previously obtained from [sqlite3_value_dup()].  ^If V is a NULL pointer
@@ -5605,7 +5677,7 @@ SQLITE_API void sqlite3_value_free(sqlite3_value*);
 **
 ** ^The sqlite3_aggregate_context(C,N) routine returns a NULL pointer
 ** when first called if N is less than or equal to zero or if a memory
-** allocate error occurs.
+** allocation error occurs.
 **
 ** ^(The amount of space allocated by sqlite3_aggregate_context(C,N) is
 ** determined by the N parameter on first successful call.  Changing the
@@ -5810,9 +5882,10 @@ typedef void (*sqlite3_destructor_type)(void*);
 ** of [SQLITE_UTF8], [SQLITE_UTF16], [SQLITE_UTF16BE], or [SQLITE_UTF16LE].
 ** ^SQLite takes the text result from the application from
 ** the 2nd parameter of the sqlite3_result_text* interfaces.
-** ^If the 3rd parameter to the sqlite3_result_text* interfaces
-** is negative, then SQLite takes result text from the 2nd parameter
-** through the first zero character.
+** ^If the 3rd parameter to any of the sqlite3_result_text* interfaces
+** other than sqlite3_result_text64() is negative, then SQLite computes
+** the string length itself by searching the 2nd parameter for the first
+** zero character.
 ** ^If the 3rd parameter to the sqlite3_result_text* interfaces
 ** is non-negative, then as many bytes (not characters) of the text
 ** pointed to by the 2nd parameter are taken as the application-defined
@@ -6256,6 +6329,28 @@ SQLITE_API int sqlite3_get_autocommit(sqlite3*);
 */
 SQLITE_API sqlite3 *sqlite3_db_handle(sqlite3_stmt*);
 
+/*
+** CAPI3REF: Return The Schema Name For A Database Connection
+** METHOD: sqlite3
+**
+** ^The sqlite3_db_name(D,N) interface returns a pointer to the schema name
+** for the N-th database on database connection D, or a NULL pointer of N is
+** out of range.  An N value of 0 means the main database file.  An N of 1 is
+** the "temp" schema.  Larger values of N correspond to various ATTACH-ed
+** databases.
+**
+** Space to hold the string that is returned by sqlite3_db_name() is managed
+** by SQLite itself.  The string might be deallocated by any operation that
+** changes the schema, including [ATTACH] or [DETACH] or calls to
+** [sqlite3_serialize()] or [sqlite3_deserialize()], even operations that
+** occur on a different thread.  Applications that need to
+** remember the string long-term should make their own copy.  Applications that
+** are accessing the same database connection simultaneously on multiple
+** threads should mutex-protect calls to this API and should make their own
+** private copy of the result prior to releasing the mutex.
+*/
+SQLITE_API const char *sqlite3_db_name(sqlite3 *db, int N);
+
 /*
 ** CAPI3REF: Return The Filename For A Database Connection
 ** METHOD: sqlite3
@@ -6286,7 +6381,7 @@ SQLITE_API sqlite3 *sqlite3_db_handle(sqlite3_stmt*);
 ** <li> [sqlite3_filename_wal()]
 ** </ul>
 */
-SQLITE_API const char *sqlite3_db_filename(sqlite3 *db, const char *zDbName);
+SQLITE_API sqlite3_filename sqlite3_db_filename(sqlite3 *db, const char *zDbName);
 
 /*
 ** CAPI3REF: Determine if a database is read-only
@@ -6423,7 +6518,7 @@ SQLITE_API void *sqlite3_rollback_hook(sqlite3*, void(*)(void *), void*);
 ** function C that is invoked prior to each autovacuum of the database
 ** file.  ^The callback is passed a copy of the generic data pointer (P),
 ** the schema-name of the attached database that is being autovacuumed,
-** the the size of the database file in pages, the number of free pages,
+** the size of the database file in pages, the number of free pages,
 ** and the number of bytes per page, respectively.  The callback should
 ** return the number of free pages that should be removed by the
 ** autovacuum.  ^If the callback returns zero, then no autovacuum happens.
@@ -6544,6 +6639,11 @@ SQLITE_API void *sqlite3_update_hook(
 ** to the same database. Sharing is enabled if the argument is true
 ** and disabled if the argument is false.)^
 **
+** This interface is omitted if SQLite is compiled with
+** [-DSQLITE_OMIT_SHARED_CACHE].  The [-DSQLITE_OMIT_SHARED_CACHE]
+** compile-time option is recommended because the
+** [use of shared cache mode is discouraged].
+**
 ** ^Cache sharing is enabled and disabled for an entire process.
 ** This is a change as of SQLite [version 3.5.0] ([dateof:3.5.0]).
 ** In prior versions of SQLite,
@@ -6642,7 +6742,7 @@ SQLITE_API int sqlite3_db_release_memory(sqlite3*);
 ** ^The soft heap limit may not be greater than the hard heap limit.
 ** ^If the hard heap limit is enabled and if sqlite3_soft_heap_limit(N)
 ** is invoked with a value of N that is greater than the hard heap limit,
-** the the soft heap limit is set to the value of the hard heap limit.
+** the soft heap limit is set to the value of the hard heap limit.
 ** ^The soft heap limit is automatically enabled whenever the hard heap
 ** limit is enabled. ^When sqlite3_hard_heap_limit64(N) is invoked and
 ** the soft heap limit is outside the range of 1..N, then the soft heap
@@ -7122,24 +7222,56 @@ struct sqlite3_index_info {
 **
 ** These macros define the allowed values for the
 ** [sqlite3_index_info].aConstraint[].op field.  Each value represents
-** an operator that is part of a constraint term in the wHERE clause of
+** an operator that is part of a constraint term in the WHERE clause of
 ** a query that uses a [virtual table].
+**
+** ^The left-hand operand of the operator is given by the corresponding
+** aConstraint[].iColumn field.  ^An iColumn of -1 indicates the left-hand
+** operand is the rowid.
+** The SQLITE_INDEX_CONSTRAINT_LIMIT and SQLITE_INDEX_CONSTRAINT_OFFSET
+** operators have no left-hand operand, and so for those operators the
+** corresponding aConstraint[].iColumn is meaningless and should not be
+** used.
+**
+** All operator values from SQLITE_INDEX_CONSTRAINT_FUNCTION through
+** value 255 are reserved to represent functions that are overloaded
+** by the [xFindFunction|xFindFunction method] of the virtual table
+** implementation.
+**
+** The right-hand operands for each constraint might be accessible using
+** the [sqlite3_vtab_rhs_value()] interface.  Usually the right-hand
+** operand is only available if it appears as a single constant literal
+** in the input SQL.  If the right-hand operand is another column or an
+** expression (even a constant expression) or a parameter, then the
+** sqlite3_vtab_rhs_value() probably will not be able to extract it.
+** ^The SQLITE_INDEX_CONSTRAINT_ISNULL and
+** SQLITE_INDEX_CONSTRAINT_ISNOTNULL operators have no right-hand operand
+** and hence calls to sqlite3_vtab_rhs_value() for those operators will
+** always return SQLITE_NOTFOUND.
+**
+** The collating sequence to be used for comparison can be found using
+** the [sqlite3_vtab_collation()] interface.  For most real-world virtual
+** tables, the collating sequence of constraints does not matter (for example
+** because the constraints are numeric) and so the sqlite3_vtab_collation()
+** interface is no commonly needed.
 */
-#define SQLITE_INDEX_CONSTRAINT_EQ         2
-#define SQLITE_INDEX_CONSTRAINT_GT         4
-#define SQLITE_INDEX_CONSTRAINT_LE         8
-#define SQLITE_INDEX_CONSTRAINT_LT        16
-#define SQLITE_INDEX_CONSTRAINT_GE        32
-#define SQLITE_INDEX_CONSTRAINT_MATCH     64
-#define SQLITE_INDEX_CONSTRAINT_LIKE      65
-#define SQLITE_INDEX_CONSTRAINT_GLOB      66
-#define SQLITE_INDEX_CONSTRAINT_REGEXP    67
-#define SQLITE_INDEX_CONSTRAINT_NE        68
-#define SQLITE_INDEX_CONSTRAINT_ISNOT     69
-#define SQLITE_INDEX_CONSTRAINT_ISNOTNULL 70
-#define SQLITE_INDEX_CONSTRAINT_ISNULL    71
-#define SQLITE_INDEX_CONSTRAINT_IS        72
-#define SQLITE_INDEX_CONSTRAINT_FUNCTION 150
+#define SQLITE_INDEX_CONSTRAINT_EQ          2
+#define SQLITE_INDEX_CONSTRAINT_GT          4
+#define SQLITE_INDEX_CONSTRAINT_LE          8
+#define SQLITE_INDEX_CONSTRAINT_LT         16
+#define SQLITE_INDEX_CONSTRAINT_GE         32
+#define SQLITE_INDEX_CONSTRAINT_MATCH      64
+#define SQLITE_INDEX_CONSTRAINT_LIKE       65
+#define SQLITE_INDEX_CONSTRAINT_GLOB       66
+#define SQLITE_INDEX_CONSTRAINT_REGEXP     67
+#define SQLITE_INDEX_CONSTRAINT_NE         68
+#define SQLITE_INDEX_CONSTRAINT_ISNOT      69
+#define SQLITE_INDEX_CONSTRAINT_ISNOTNULL  70
+#define SQLITE_INDEX_CONSTRAINT_ISNULL     71
+#define SQLITE_INDEX_CONSTRAINT_IS         72
+#define SQLITE_INDEX_CONSTRAINT_LIMIT      73
+#define SQLITE_INDEX_CONSTRAINT_OFFSET     74
+#define SQLITE_INDEX_CONSTRAINT_FUNCTION  150
 
 /*
 ** CAPI3REF: Register A Virtual Table Implementation
@@ -7168,7 +7300,7 @@ struct sqlite3_index_info {
 ** destructor.
 **
 ** ^If the third parameter (the pointer to the sqlite3_module object) is
-** NULL then no new module is create and any existing modules with the
+** NULL then no new module is created and any existing modules with the
 ** same name are dropped.
 **
 ** See also: [sqlite3_drop_modules()]
@@ -7944,7 +8076,8 @@ SQLITE_API int sqlite3_test_control(int op, ...);
 #define SQLITE_TESTCTRL_SEEK_COUNT              30
 #define SQLITE_TESTCTRL_TRACEFLAGS              31
 #define SQLITE_TESTCTRL_TUNE                    32
-#define SQLITE_TESTCTRL_LAST                    32  /* Largest TESTCTRL */
+#define SQLITE_TESTCTRL_LOGEST                  33
+#define SQLITE_TESTCTRL_LAST                    33  /* Largest TESTCTRL */
 
 /*
 ** CAPI3REF: SQL Keyword Checking
@@ -8467,6 +8600,16 @@ SQLITE_API int sqlite3_stmt_status(sqlite3_stmt*, int op,int resetFlg);
 ** The counter is incremented on the first [sqlite3_step()] call of each
 ** cycle.
 **
+** [[SQLITE_STMTSTATUS_FILTER_MISS]]
+** [[SQLITE_STMTSTATUS_FILTER HIT]]
+** <dt>SQLITE_STMTSTATUS_FILTER_HIT<br>
+** SQLITE_STMTSTATUS_FILTER_MISS</dt>
+** <dd>^SQLITE_STMTSTATUS_FILTER_HIT is the number of times that a join
+** step was bypassed because a Bloom filter returned not-found.  The
+** corresponding SQLITE_STMTSTATUS_FILTER_MISS value is the number of
+** times that the Bloom filter returned a find, and thus the join step
+** had to be processed as normal.
+**
 ** [[SQLITE_STMTSTATUS_MEMUSED]] <dt>SQLITE_STMTSTATUS_MEMUSED</dt>
 ** <dd>^This is the approximate number of bytes of heap memory
 ** used to store the prepared statement.  ^This value is not actually
@@ -8481,6 +8624,8 @@ SQLITE_API int sqlite3_stmt_status(sqlite3_stmt*, int op,int resetFlg);
 #define SQLITE_STMTSTATUS_VM_STEP           4
 #define SQLITE_STMTSTATUS_REPREPARE         5
 #define SQLITE_STMTSTATUS_RUN               6
+#define SQLITE_STMTSTATUS_FILTER_MISS       7
+#define SQLITE_STMTSTATUS_FILTER_HIT        8
 #define SQLITE_STMTSTATUS_MEMUSED           99
 
 /*
@@ -8892,7 +9037,7 @@ typedef struct sqlite3_backup sqlite3_backup;
 ** if the application incorrectly accesses the destination [database connection]
 ** and so no error code is reported, but the operations may malfunction
 ** nevertheless.  Use of the destination database connection while a
-** backup is in progress might also also cause a mutex deadlock.
+** backup is in progress might also cause a mutex deadlock.
 **
 ** If running in [shared cache mode], the application must
 ** guarantee that the shared cache used by the destination database
@@ -9320,7 +9465,7 @@ SQLITE_API int sqlite3_wal_checkpoint_v2(
 */
 #define SQLITE_CHECKPOINT_PASSIVE  0  /* Do as much as possible w/o blocking */
 #define SQLITE_CHECKPOINT_FULL     1  /* Wait for writers, then checkpoint */
-#define SQLITE_CHECKPOINT_RESTART  2  /* Like FULL but wait for for readers */
+#define SQLITE_CHECKPOINT_RESTART  2  /* Like FULL but wait for readers */
 #define SQLITE_CHECKPOINT_TRUNCATE 3  /* Like RESTART but also truncate WAL */
 
 /*
@@ -9449,19 +9594,276 @@ SQLITE_API int sqlite3_vtab_nochange(sqlite3_context*);
 
 /*
 ** CAPI3REF: Determine The Collation For a Virtual Table Constraint
+** METHOD: sqlite3_index_info
 **
 ** This function may only be called from within a call to the [xBestIndex]
-** method of a [virtual table].
+** method of a [virtual table].  This function returns a pointer to a string
+** that is the name of the appropriate collation sequence to use for text
+** comparisons on the constraint identified by its arguments.
 **
-** The first argument must be the sqlite3_index_info object that is the
-** first parameter to the xBestIndex() method. The second argument must be
-** an index into the aConstraint[] array belonging to the sqlite3_index_info
-** structure passed to xBestIndex. This function returns a pointer to a buffer
-** containing the name of the collation sequence for the corresponding
-** constraint.
+** The first argument must be the pointer to the [sqlite3_index_info] object
+** that is the first parameter to the xBestIndex() method. The second argument
+** must be an index into the aConstraint[] array belonging to the
+** sqlite3_index_info structure passed to xBestIndex.
+**
+** Important:
+** The first parameter must be the same pointer that is passed into the
+** xBestMethod() method.  The first parameter may not be a pointer to a
+** different [sqlite3_index_info] object, even an exact copy.
+**
+** The return value is computed as follows:
+**
+** <ol>
+** <li><p> If the constraint comes from a WHERE clause expression that contains
+**         a [COLLATE operator], then the name of the collation specified by
+**         that COLLATE operator is returned.
+** <li><p> If there is no COLLATE operator, but the column that is the subject
+**         of the constraint specifies an alternative collating sequence via
+**         a [COLLATE clause] on the column definition within the CREATE TABLE
+**         statement that was passed into [sqlite3_declare_vtab()], then the
+**         name of that alternative collating sequence is returned.
+** <li><p> Otherwise, "BINARY" is returned.
+** </ol>
 */
 SQLITE_API SQLITE_EXPERIMENTAL const char *sqlite3_vtab_collation(sqlite3_index_info*,int);
 
+/*
+** CAPI3REF: Determine if a virtual table query is DISTINCT
+** METHOD: sqlite3_index_info
+**
+** This API may only be used from within an [xBestIndex|xBestIndex method]
+** of a [virtual table] implementation. The result of calling this
+** interface from outside of xBestIndex() is undefined and probably harmful.
+**
+** ^The sqlite3_vtab_distinct() interface returns an integer between 0 and
+** 3.  The integer returned by sqlite3_vtab_distinct()
+** gives the virtual table additional information about how the query
+** planner wants the output to be ordered. As long as the virtual table
+** can meet the ordering requirements of the query planner, it may set
+** the "orderByConsumed" flag.
+**
+** <ol><li value="0"><p>
+** ^If the sqlite3_vtab_distinct() interface returns 0, that means
+** that the query planner needs the virtual table to return all rows in the
+** sort order defined by the "nOrderBy" and "aOrderBy" fields of the
+** [sqlite3_index_info] object.  This is the default expectation.  If the
+** virtual table outputs all rows in sorted order, then it is always safe for
+** the xBestIndex method to set the "orderByConsumed" flag, regardless of
+** the return value from sqlite3_vtab_distinct().
+** <li value="1"><p>
+** ^(If the sqlite3_vtab_distinct() interface returns 1, that means
+** that the query planner does not need the rows to be returned in sorted order
+** as long as all rows with the same values in all columns identified by the
+** "aOrderBy" field are adjacent.)^  This mode is used when the query planner
+** is doing a GROUP BY.
+** <li value="2"><p>
+** ^(If the sqlite3_vtab_distinct() interface returns 2, that means
+** that the query planner does not need the rows returned in any particular
+** order, as long as rows with the same values in all "aOrderBy" columns
+** are adjacent.)^  ^(Furthermore, only a single row for each particular
+** combination of values in the columns identified by the "aOrderBy" field
+** needs to be returned.)^  ^It is always ok for two or more rows with the same
+** values in all "aOrderBy" columns to be returned, as long as all such rows
+** are adjacent.  ^The virtual table may, if it chooses, omit extra rows
+** that have the same value for all columns identified by "aOrderBy".
+** ^However omitting the extra rows is optional.
+** This mode is used for a DISTINCT query.
+** <li value="3"><p>
+** ^(If the sqlite3_vtab_distinct() interface returns 3, that means
+** that the query planner needs only distinct rows but it does need the
+** rows to be sorted.)^ ^The virtual table implementation is free to omit
+** rows that are identical in all aOrderBy columns, if it wants to, but
+** it is not required to omit any rows.  This mode is used for queries
+** that have both DISTINCT and ORDER BY clauses.
+** </ol>
+**
+** ^For the purposes of comparing virtual table output values to see if the
+** values are same value for sorting purposes, two NULL values are considered
+** to be the same.  In other words, the comparison operator is "IS"
+** (or "IS NOT DISTINCT FROM") and not "==".
+**
+** If a virtual table implementation is unable to meet the requirements
+** specified above, then it must not set the "orderByConsumed" flag in the
+** [sqlite3_index_info] object or an incorrect answer may result.
+**
+** ^A virtual table implementation is always free to return rows in any order
+** it wants, as long as the "orderByConsumed" flag is not set.  ^When the
+** the "orderByConsumed" flag is unset, the query planner will add extra
+** [bytecode] to ensure that the final results returned by the SQL query are
+** ordered correctly.  The use of the "orderByConsumed" flag and the
+** sqlite3_vtab_distinct() interface is merely an optimization.  ^Careful
+** use of the sqlite3_vtab_distinct() interface and the "orderByConsumed"
+** flag might help queries against a virtual table to run faster.  Being
+** overly aggressive and setting the "orderByConsumed" flag when it is not
+** valid to do so, on the other hand, might cause SQLite to return incorrect
+** results.
+*/
+SQLITE_API int sqlite3_vtab_distinct(sqlite3_index_info*);
+
+/*
+** CAPI3REF: Identify and handle IN constraints in xBestIndex
+**
+** This interface may only be used from within an
+** [xBestIndex|xBestIndex() method] of a [virtual table] implementation.
+** The result of invoking this interface from any other context is
+** undefined and probably harmful.
+**
+** ^(A constraint on a virtual table of the form
+** "[IN operator|column IN (...)]" is
+** communicated to the xBestIndex method as a
+** [SQLITE_INDEX_CONSTRAINT_EQ] constraint.)^  If xBestIndex wants to use
+** this constraint, it must set the corresponding
+** aConstraintUsage[].argvIndex to a postive integer.  ^(Then, under
+** the usual mode of handling IN operators, SQLite generates [bytecode]
+** that invokes the [xFilter|xFilter() method] once for each value
+** on the right-hand side of the IN operator.)^  Thus the virtual table
+** only sees a single value from the right-hand side of the IN operator
+** at a time.
+**
+** In some cases, however, it would be advantageous for the virtual
+** table to see all values on the right-hand of the IN operator all at
+** once.  The sqlite3_vtab_in() interfaces facilitates this in two ways:
+**
+** <ol>
+** <li><p>
+**   ^A call to sqlite3_vtab_in(P,N,-1) will return true (non-zero)
+**   if and only if the [sqlite3_index_info|P->aConstraint][N] constraint
+**   is an [IN operator] that can be processed all at once.  ^In other words,
+**   sqlite3_vtab_in() with -1 in the third argument is a mechanism
+**   by which the virtual table can ask SQLite if all-at-once processing
+**   of the IN operator is even possible.
+**
+** <li><p>
+**   ^A call to sqlite3_vtab_in(P,N,F) with F==1 or F==0 indicates
+**   to SQLite that the virtual table does or does not want to process
+**   the IN operator all-at-once, respectively.  ^Thus when the third
+**   parameter (F) is non-negative, this interface is the mechanism by
+**   which the virtual table tells SQLite how it wants to process the
+**   IN operator.
+** </ol>
+**
+** ^The sqlite3_vtab_in(P,N,F) interface can be invoked multiple times
+** within the same xBestIndex method call.  ^For any given P,N pair,
+** the return value from sqlite3_vtab_in(P,N,F) will always be the same
+** within the same xBestIndex call.  ^If the interface returns true
+** (non-zero), that means that the constraint is an IN operator
+** that can be processed all-at-once.  ^If the constraint is not an IN
+** operator or cannot be processed all-at-once, then the interface returns
+** false.
+**
+** ^(All-at-once processing of the IN operator is selected if both of the
+** following conditions are met:
+**
+** <ol>
+** <li><p> The P->aConstraintUsage[N].argvIndex value is set to a positive
+** integer.  This is how the virtual table tells SQLite that it wants to
+** use the N-th constraint.
+**
+** <li><p> The last call to sqlite3_vtab_in(P,N,F) for which F was
+** non-negative had F>=1.
+** </ol>)^
+**
+** ^If either or both of the conditions above are false, then SQLite uses
+** the traditional one-at-a-time processing strategy for the IN constraint.
+** ^If both conditions are true, then the argvIndex-th parameter to the
+** xFilter method will be an [sqlite3_value] that appears to be NULL,
+** but which can be passed to [sqlite3_vtab_in_first()] and
+** [sqlite3_vtab_in_next()] to find all values on the right-hand side
+** of the IN constraint.
+*/
+SQLITE_API int sqlite3_vtab_in(sqlite3_index_info*, int iCons, int bHandle);
+
+/*
+** CAPI3REF: Find all elements on the right-hand side of an IN constraint.
+**
+** These interfaces are only useful from within the
+** [xFilter|xFilter() method] of a [virtual table] implementation.
+** The result of invoking these interfaces from any other context
+** is undefined and probably harmful.
+**
+** The X parameter in a call to sqlite3_vtab_in_first(X,P) or
+** sqlite3_vtab_in_next(X,P) must be one of the parameters to the
+** xFilter method which invokes these routines, and specifically
+** a parameter that was previously selected for all-at-once IN constraint
+** processing use the [sqlite3_vtab_in()] interface in the
+** [xBestIndex|xBestIndex method].  ^(If the X parameter is not
+** an xFilter argument that was selected for all-at-once IN constraint
+** processing, then these routines return [SQLITE_MISUSE])^ or perhaps
+** exhibit some other undefined or harmful behavior.
+**
+** ^(Use these routines to access all values on the right-hand side
+** of the IN constraint using code like the following:
+**
+** <blockquote><pre>
+** &nbsp;  for(rc=sqlite3_vtab_in_first(pList, &pVal);
+** &nbsp;      rc==SQLITE_OK && pVal
+** &nbsp;      rc=sqlite3_vtab_in_next(pList, &pVal)
+** &nbsp;  ){
+** &nbsp;    // do something with pVal
+** &nbsp;  }
+** &nbsp;  if( rc!=SQLITE_OK ){
+** &nbsp;    // an error has occurred
+** &nbsp;  }
+** </pre></blockquote>)^
+**
+** ^On success, the sqlite3_vtab_in_first(X,P) and sqlite3_vtab_in_next(X,P)
+** routines return SQLITE_OK and set *P to point to the first or next value
+** on the RHS of the IN constraint.  ^If there are no more values on the
+** right hand side of the IN constraint, then *P is set to NULL and these
+** routines return [SQLITE_DONE].  ^The return value might be
+** some other value, such as SQLITE_NOMEM, in the event of a malfunction.
+**
+** The *ppOut values returned by these routines are only valid until the
+** next call to either of these routines or until the end of the xFilter
+** method from which these routines were called.  If the virtual table
+** implementation needs to retain the *ppOut values for longer, it must make
+** copies.  The *ppOut values are [protected sqlite3_value|protected].
+*/
+SQLITE_API int sqlite3_vtab_in_first(sqlite3_value *pVal, sqlite3_value **ppOut);
+SQLITE_API int sqlite3_vtab_in_next(sqlite3_value *pVal, sqlite3_value **ppOut);
+
+/*
+** CAPI3REF: Constraint values in xBestIndex()
+** METHOD: sqlite3_index_info
+**
+** This API may only be used from within the [xBestIndex|xBestIndex method]
+** of a [virtual table] implementation. The result of calling this interface
+** from outside of an xBestIndex method are undefined and probably harmful.
+**
+** ^When the sqlite3_vtab_rhs_value(P,J,V) interface is invoked from within
+** the [xBestIndex] method of a [virtual table] implementation, with P being
+** a copy of the [sqlite3_index_info] object pointer passed into xBestIndex and
+** J being a 0-based index into P->aConstraint[], then this routine
+** attempts to set *V to the value of the right-hand operand of
+** that constraint if the right-hand operand is known.  ^If the
+** right-hand operand is not known, then *V is set to a NULL pointer.
+** ^The sqlite3_vtab_rhs_value(P,J,V) interface returns SQLITE_OK if
+** and only if *V is set to a value.  ^The sqlite3_vtab_rhs_value(P,J,V)
+** inteface returns SQLITE_NOTFOUND if the right-hand side of the J-th
+** constraint is not available.  ^The sqlite3_vtab_rhs_value() interface
+** can return an result code other than SQLITE_OK or SQLITE_NOTFOUND if
+** something goes wrong.
+**
+** The sqlite3_vtab_rhs_value() interface is usually only successful if
+** the right-hand operand of a constraint is a literal value in the original
+** SQL statement.  If the right-hand operand is an expression or a reference
+** to some other column or a [host parameter], then sqlite3_vtab_rhs_value()
+** will probably return [SQLITE_NOTFOUND].
+**
+** ^(Some constraints, such as [SQLITE_INDEX_CONSTRAINT_ISNULL] and
+** [SQLITE_INDEX_CONSTRAINT_ISNOTNULL], have no right-hand operand.  For such
+** constraints, sqlite3_vtab_rhs_value() always returns SQLITE_NOTFOUND.)^
+**
+** ^The [sqlite3_value] object returned in *V is a protected sqlite3_value
+** and remains valid for the duration of the xBestIndex method call.
+** ^When xBestIndex returns, the sqlite3_value object returned by
+** sqlite3_vtab_rhs_value() is automatically deallocated.
+**
+** The "_rhs_" in the name of this routine is an abbreviation for
+** "Right-Hand Side".
+*/
+SQLITE_API int sqlite3_vtab_rhs_value(sqlite3_index_info*, int, sqlite3_value **ppVal);
+
 /*
 ** CAPI3REF: Conflict resolution modes
 ** KEYWORDS: {conflict resolution mode}

3party/sqlite3/sqlite3ext.h

@@ -331,9 +331,9 @@ struct sqlite3_api_routines {
   const char *(*filename_journal)(const char*);
   const char *(*filename_wal)(const char*);
   /* Version 3.32.0 and later */
-  char *(*create_filename)(const char*,const char*,const char*,
+  const char *(*create_filename)(const char*,const char*,const char*,
                            int,const char**);
-  void (*free_filename)(char*);
+  void (*free_filename)(const char*);
   sqlite3_file *(*database_file_object)(const char*);
   /* Version 3.34.0 and later */
   int (*txn_state)(sqlite3*,const char*);
@@ -344,6 +344,21 @@ struct sqlite3_api_routines {
   int (*autovacuum_pages)(sqlite3*,
      unsigned int(*)(void*,const char*,unsigned int,unsigned int,unsigned int),
      void*, void(*)(void*));
+  /* Version 3.38.0 and later */
+  int (*error_offset)(sqlite3*);
+  int (*vtab_rhs_value)(sqlite3_index_info*,int,sqlite3_value**);
+  int (*vtab_distinct)(sqlite3_index_info*);
+  int (*vtab_in)(sqlite3_index_info*,int,int);
+  int (*vtab_in_first)(sqlite3_value*,sqlite3_value**);
+  int (*vtab_in_next)(sqlite3_value*,sqlite3_value**);
+  /* Version 3.39.0 and later */
+  int (*deserialize)(sqlite3*,const char*,unsigned char*,
+                     sqlite3_int64,sqlite3_int64,unsigned);
+  unsigned char *(*serialize)(sqlite3*,const char *,sqlite3_int64*,
+                              unsigned int);
+  const char *(*db_name)(sqlite3*,int);
+  /* Version 3.40.0 and later */
+  int (*value_encoding)(sqlite3_value*);
 };
 
 /*
@@ -655,6 +670,21 @@ typedef int (*sqlite3_loadext_entry)(
 #define sqlite3_total_changes64        sqlite3_api->total_changes64
 /* Version 3.37.0 and later */
 #define sqlite3_autovacuum_pages       sqlite3_api->autovacuum_pages
+/* Version 3.38.0 and later */
+#define sqlite3_error_offset           sqlite3_api->error_offset
+#define sqlite3_vtab_rhs_value         sqlite3_api->vtab_rhs_value
+#define sqlite3_vtab_distinct          sqlite3_api->vtab_distinct
+#define sqlite3_vtab_in                sqlite3_api->vtab_in
+#define sqlite3_vtab_in_first          sqlite3_api->vtab_in_first
+#define sqlite3_vtab_in_next           sqlite3_api->vtab_in_next
+/* Version 3.39.0 and later */
+#ifndef SQLITE_OMIT_DESERIALIZE
+#define sqlite3_deserialize            sqlite3_api->deserialize
+#define sqlite3_serialize              sqlite3_api->serialize
+#endif
+#define sqlite3_db_name                sqlite3_api->db_name
+/* Version 3.40.0 and later */
+#define sqlite3_value_encoding         sqlite3_api->value_encoding
 #endif /* !defined(SQLITE_CORE) && !defined(SQLITE_OMIT_LOAD_EXTENSION) */
 
 #if !defined(SQLITE_CORE) && !defined(SQLITE_OMIT_LOAD_EXTENSION)

CMakeLists.txt

@@ -14,108 +14,18 @@ if( CMAKE_GENERATOR MATCHES "Visual Studio" )
     )
 endif()
 
-list(APPEND CMAKE_MODULE_PATH ${CMAKE_BINARY_DIR})
-list(APPEND CMAKE_PREFIX_PATH ${CMAKE_BINARY_DIR})
-
-if(NOT EXISTS "${CMAKE_BINARY_DIR}/conan.cmake")
-    message(STATUS "Downloading conan.cmake from https://github.com/conan-io/cmake-conan")
-    file(DOWNLOAD "https://raw.githubusercontent.com/conan-io/cmake-conan/master/conan.cmake"
-            "${CMAKE_BINARY_DIR}/conan.cmake"
-            TLS_VERIFY ON)
-endif()
-
-include(${CMAKE_BINARY_DIR}/conan.cmake)
-
 set(CMAKE_CXX_STANDARD 17)
 
 if(WIN32)
     add_definitions(-DUNICODE -D_UNICODE)
 endif()
 
-set(DEPENDENCIES
-    REQUIRES
-        fmt/7.1.3
-        sqlite3/3.37.0
-        sqlitecpp/3.1.1
-        gflags/2.2.2
-        utfcpp/3.2.1
-        boost/1.78.0
-
-    OPTIONS
-        boost:without_atomic=False
-        boost:without_chrono=True
-        boost:without_container=True
-        boost:without_context=True
-        boost:without_contract=True
-        boost:without_coroutine=True
-        boost:without_date_time=True
-        boost:without_exception=True
-        boost:without_fiber=True
-        boost:without_filesystem=False
-        boost:without_graph=True
-        boost:without_graph_parallel=True
-        boost:without_iostreams=True
-        boost:without_json=True
-        boost:without_locale=True
-        boost:without_log=True
-        boost:without_math=True
-        boost:without_mpi=True
-        boost:without_nowide=True
-        boost:without_program_options=True
-        boost:without_python=True
-        boost:without_random=True
-        boost:without_regex=True
-        boost:without_serialization=True
-        boost:without_stacktrace=True
-        boost:without_system=False
-        boost:without_test=True
-        boost:without_thread=True
-        boost:without_timer=True
-        boost:without_type_erasure=True
-        boost:without_wave=True
-)
-
-### Setup dependecies
-if(CMAKE_CONFIGURATION_TYPES)
-    foreach(TYPE ${CMAKE_CONFIGURATION_TYPES})
-        message(STATUS "Configuring packages for ${TYPE}")
-
-        conan_cmake_configure(
-            ${DEPENDENCIES}
-            GENERATORS
-                cmake_find_package_multi
-        )
-
-        conan_cmake_autodetect(settings BUILD_TYPE ${TYPE})
-
-        conan_cmake_install(PATH_OR_REFERENCE .
-            BUILD missing
-            SETTINGS ${settings}
-        )
-    endforeach()
-else()
-    message(STATUS "Configuring packages for ${CMAKE_BUILD_TYPE}")
-
-    conan_cmake_configure(
-        ${DEPENDENCIES}
-        GENERATORS
-            cmake_find_package_multi
-    )
-
-    conan_cmake_autodetect(settings)
-
-    conan_cmake_install(PATH_OR_REFERENCE .
-        BUILD missing
-        SETTINGS ${settings}
-    )
-endif()
-
-find_package(fmt CONFIG)
-find_package(SQLite3 CONFIG)
-find_package(SQLiteCpp CONFIG)
-find_package(gflags CONFIG)
-find_package(utf8cpp CONFIG)
-find_package(Boost)
+find_package(fmt REQUIRED)
+find_package(SQLite3 REQUIRED)
+find_package(SQLiteCpp REQUIRED)
+find_package(gflags REQUIRED)
+find_package(utf8cpp REQUIRED)
+find_package(Boost REQUIRED COMPONENTS filesystem system)
 
 set( CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin )
 
@@ -150,9 +60,9 @@ add_executable(audacity-project-tools
 target_link_libraries(audacity-project-tools
     fmt::fmt
     SQLite::SQLite3
-    SQLiteCpp::SQLiteCpp
+    SQLiteCpp
     gflags::gflags
-    utf8cpp::utf8cpp
+    utf8cpp
     Boost::boost
     Boost::filesystem
     Boost::system
@@ -166,7 +76,7 @@ set(CPACK_PACKAGE_NAME "audacity-project-tools")
 set(CPACK_PACKAGE_VENDOR "Audacity")
 set(CPACK_PACKAGE_VERSION_MAJOR 1)
 set(CPACK_PACKAGE_VERSION_MINOR 0)
-set(CPACK_PACKAGE_VERSION_PATCH 0)
+set(CPACK_PACKAGE_VERSION_PATCH 5)
 set(CPACK_GENERATOR ZIP)
 SET(CPACK_OUTPUT_FILE_PREFIX packages)
 

README.md

@@ -33,6 +33,7 @@ Audacity stores samples in blocks of up to 1Mb inside the `sampleblocks` table.
 * `-extract_sample_blocks`: extract sample blocks as separate wav files. It can be used if the project table is corrupted.
 * `-extract_as_mono_track`: extract sample blocks as a single mono wav file.
 * `-extract_as_stereo_track`: extract sample blocks as a single stereo wav file. Channels are based on the parity of the block_id.
+* `-analyze_project`: prints information about tracks and clips in the project.
 
 `audacity-project-tools` will never modify the original file. If mode requires the modification of the database, the tool will create a copy. All the output goes to the same directory as the project file has.
 
@@ -47,4 +48,4 @@ $ ls
 
 `audacity-project-tools` requires a C++17 compliant compiler with the complete C++17 library. The build requires <filesystem> and floating-point versions of `from_chars`.
 
-CMake and Conan are required to configure the project.
+CMake and Conan are required to configure the project. Conan 2.0 is not supported ATM.

conanfile.txt

@@ -0,0 +1,44 @@
+[requires]
+fmt/7.1.3
+sqlite3/3.45.3
+sqlitecpp/3.3.3
+gflags/2.2.2
+utfcpp/3.2.1
+boost/1.84.0
+
+[generators]
+CMakeDeps
+CMakeToolchain
+
+[options]
+boost/*:without_atomic=False
+boost/*:without_chrono=True
+boost/*:without_container=True
+boost/*:without_context=True
+boost/*:without_contract=True
+boost/*:without_coroutine=True
+boost/*:without_date_time=True
+boost/*:without_exception=True
+boost/*:without_fiber=True
+boost/*:without_filesystem=False
+boost/*:without_graph=True
+boost/*:without_graph_parallel=True
+boost/*:without_iostreams=True
+boost/*:without_json=True
+boost/*:without_locale=True
+boost/*:without_log=True
+boost/*:without_math=True
+boost/*:without_mpi=True
+boost/*:without_nowide=True
+boost/*:without_program_options=True
+boost/*:without_python=True
+boost/*:without_random=True
+boost/*:without_regex=True
+boost/*:without_serialization=True
+boost/*:without_stacktrace=True
+boost/*:without_system=False
+boost/*:without_test=True
+boost/*:without_thread=True
+boost/*:without_timer=True
+boost/*:without_type_erasure=True
+boost/*:without_wave=True

requirements.txt

@@ -1,21 +1 @@
-bottle==0.12.19
-certifi==2021.10.8
-charset-normalizer==2.0.9
-colorama==0.4.4
-conan==1.43.2
-distro==1.6.0
-fasteners==0.16.3
-idna==3.3
-Jinja2==2.11.3
-MarkupSafe==2.0.1
-node-semver==0.6.1
-patch-ng==1.17.4
-pluginbase==1.0.1
-Pygments==2.10.0
-PyJWT==1.7.1
-python-dateutil==2.8.2
-PyYAML==5.4.1
-requests==2.26.0
-six==1.16.0
-tqdm==4.62.3
-urllib3==1.26.7
+conan>=2

src/AudacityDatabase.cpp

@@ -25,7 +25,7 @@ namespace
 
 constexpr int64_t AudacityProjectID = 1096107097;
 constexpr int64_t MaxSupportedVersion =
-    (3 << 24) + (1 << 16) + (3 << 8) + 0; // 3.1.3.0
+    (3 << 24) + (7 << 16) + (0 << 8) + 0; // 3.7.0.0
 
 template<typename T> size_t readInt(const std::string& string, size_t offset, T& output)
 {
@@ -112,16 +112,32 @@ AudacityDatabase::AudacityDatabase(
             mProjectVersion =
                 mDatabase->execAndGet("PRAGMA user_version;").getUInt();
 
+            const auto majorVersion = (mProjectVersion >> 24) & 0xFF;
+            const auto minorVersion = (mProjectVersion >> 16) & 0xFF;
+            const auto patchVersion = (mProjectVersion >> 8) & 0xFF;
+
             fmt::print(
-                "Project requires Audacity {}.{}.{}\n",
-                (mProjectVersion >> 24) & 0xFF, (mProjectVersion >> 16) & 0xFF,
-                (mProjectVersion >> 8) & 0xFF);
+                "Project requires Audacity {}.{}.{}\n", majorVersion,
+                minorVersion, patchVersion);
+
+            if (majorVersion != 3)
+            {
+                fmt::print("user_version pragma appears to be broken...\n");
+                mProjectVersion = MaxSupportedVersion;
+            }
 
             if (mProjectVersion > MaxSupportedVersion)
-                throw std::runtime_error("Unsupported project version");
+                fmt::print(
+                    "DANGER!!! Unsupported Audacity version detected! Project data might be lost! Proceed with caution!\n");
         }, true);
 }
 
+AudacityDatabase::~AudacityDatabase()
+{
+    mDatabase.reset();
+    removeJournalFiles(mReadOnly ? mProjectPath : mWritablePath);
+}
+
 void AudacityDatabase::reopenReadonlyAsWritable()
 {
     if (!mReadOnly)
@@ -177,7 +193,7 @@ void AudacityDatabase::recoverDatabase()
     child c(
         sqlite3Binary,
         args += { mProjectPath.string(),
-                  mRecoveryConfig.FreelistCorrupt ? ".recover --freelist-corrupt" : ".recover" },
+                  mRecoveryConfig.FreelistCorrupt ? ".recover --ignore-freelist" : ".recover" },
         std_out > out_stream, std_err > err_stream);
 
     std::string line;
@@ -288,8 +304,12 @@ void AudacityDatabase::recoverDatabase()
 
     if (result != 0)
     {
-        throw std::runtime_error(
-            std::string(std::istreambuf_iterator<char>(err_stream), {}));
+        const auto errorMessge =
+            std::string(std::istreambuf_iterator<char>(err_stream), {});
+
+        fmt::print("sqlite3 has failed {}: {}\n", result, errorMessge);
+        
+        //throw std::runtime_error(errorMessge);
     }
 
     recoveredDB->exec(R"(
@@ -452,22 +472,26 @@ void AudacityDatabase::extractTrack(
     waveFile.writeFile();
 }
 
+void AudacityDatabase::removeJournalFiles(const std::filesystem::path& dbPath)
+{
+    auto walFile = dbPath;
+    walFile.replace_extension("aup3-wal");
+
+    if (std::filesystem::exists(walFile))
+        std::filesystem::remove(walFile);
+
+    auto shmFile = dbPath;
+    shmFile.replace_extension("aup3-shm");
+
+    if (std::filesystem::exists(shmFile))
+        std::filesystem::remove(shmFile);
+}
+
 void AudacityDatabase::removeOldFiles()
 {
     if (std::filesystem::exists(mWritablePath))
     {
         std::filesystem::remove(mWritablePath);
-
-        auto walFile = mWritablePath;
-        walFile.replace_extension("aup3-wal");
-
-        if (std::filesystem::exists(walFile))
-            std::filesystem::remove(walFile);
-
-        auto shmFile = mWritablePath;
-        shmFile.replace_extension("aup3-shm");
-
-        if (std::filesystem::exists(shmFile))
-            std::filesystem::remove(shmFile);
+        removeJournalFiles(mWritablePath);
     }
 }

src/AudacityDatabase.h

@@ -24,6 +24,7 @@ class AudacityDatabase final
 public:
     explicit AudacityDatabase(
         const std::filesystem::path& path, RecoveryConfig recoveryConfig);
+    ~AudacityDatabase();
 
     void reopenReadonlyAsWritable();
     void recoverDatabase();
@@ -43,6 +44,7 @@ public:
     void extractTrack(SampleFormat format, int32_t sampleRate, bool asStereo);
 
 private:
+    static void removeJournalFiles(const std::filesystem::path& dbPath);
     void removeOldFiles();
 
     std::unique_ptr<SQLite::Database> mDatabase;

src/BinaryXMLConverter.cpp

@@ -281,17 +281,10 @@ class IdsLookup final
 public:
     void store(uint16_t index, std::string value)
     {
-        const auto size = mIds.size();
+        if (index >= mIds.size())
+            mIds.resize(index + 1);
 
-        if (index == size)
-            mIds.push_back(std::move(value));
-        else
-        {
-            if ((index + 1) < size)
-                mIds.resize(index);
-
-            mIds[index] = std::move(value);
-        }
+        mIds[index] = std::move(value);
     }
 
     std::string_view get(uint16_t index)

src/Buffer.cpp

@@ -86,6 +86,7 @@ Buffer::read(void* data, size_t offset, size_t size) const noexcept
         offset = 0;
         bytesLeft -= chunkSize;
         outPtr += chunkSize;
+        ++chunk;
     }
 
     return size;

src/Buffer.h

@@ -48,12 +48,12 @@ public:
             return 0;
 
         const size_t chunkIndex = offset / BUFFER_SIZE;
-        offset = offset - BUFFER_SIZE * chunkIndex;
+        const size_t chunkOffset = offset - BUFFER_SIZE * chunkIndex;
 
-        if (BUFFER_SIZE < (offset + size))
+        if (BUFFER_SIZE < (chunkOffset + size))
             return read(&data, offset, size);
 
-        const void* ptr = mChunks[chunkIndex].data() + offset;
+        const void* ptr = mChunks[chunkIndex].data() + chunkOffset;
 
         data = *static_cast<const T*>(ptr);
 

src/ProjectModel.cpp

@@ -48,6 +48,20 @@ void WaveBlock::convertToSilence() noexcept
     mXMLNode->setAttribute("badblock", true);
 }
 
+void WaveBlock::setBlockId(int64_t blockId) noexcept
+{
+    mBlockId = blockId;
+    mXMLNode->setAttribute("blockid", mBlockId);
+    mXMLNode->setAttribute("badblock", true);
+}
+
+void WaveBlock::setStart(int64_t start) noexcept
+{
+    mStart = start;
+    mXMLNode->setAttribute("start", mStart);
+    mXMLNode->setAttribute("badblock", true);
+}
+
 int64_t WaveBlock::getBlockId() const noexcept
 {
     return mBlockId;
@@ -226,6 +240,11 @@ int WaveTrack::getSampleRate() const
     return mRate;
 }
 
+const std::vector<Clip*>& WaveTrack::getClips() const
+{
+    return mClips;
+}
+
 AudacityProject::AudacityProject(AudacityDatabase& db)
     : mDb(db)
 {
@@ -245,6 +264,66 @@ AudacityProject::~AudacityProject()
 {
 }
 
+bool AudacityProject::containsBlock(int64_t blockId) const
+{
+    SQLite::Statement query(
+        mDb.DB(), "SELECT 1 FROM sampleblocks WHERE blockid = ?1;");
+
+    query.bind(1, blockId);
+
+    while (query.executeStep())
+        return true;
+
+    return false;
+}
+
+int AudacityProject::getRealBlockLength(const WaveBlock& block) const
+{
+    if (block.getBlockId() < 0)
+        return -block.getBlockId();
+
+    SQLite::Statement query (
+        mDb.DB(), "SELECT sampleformat, samples FROM sampleblocks WHERE blockid = ?1;");
+
+    query.bind(1, block.getBlockId());
+
+    while (query.executeStep ())
+    {
+        const int format = query.getColumn(0).getInt();
+        const int64_t blobSize = query.getColumn(1).getBytes();
+
+        if (format != block.getParent()->getFormat())
+            throw std::runtime_error (
+                fmt::format("Unexpected sample format for block {}",
+                                   block.getBlockId()));
+
+        return blobSize / BytesPerSample(static_cast<SampleFormat>(format));
+    }
+
+    throw std::runtime_error (
+        fmt::format("Block {} not found in the database", block.getBlockId()));
+}
+
+AudacityProject::BlockValidationResult
+AudacityProject::validateBlock(const WaveBlock& block) const
+{
+    // Creating query every time is slower,
+    // but can withstand error
+    SQLite::Statement query(
+        mDb.DB(), "SELECT sampleformat FROM sampleblocks WHERE blockid = ?1;");
+
+    query.bind(1, block.getBlockId());
+
+    while (query.executeStep())
+    {
+        const int format = query.getColumn(0).getInt();
+
+        return block.getParent()->getFormat() == format ? BlockValidationResult::Ok : BlockValidationResult::Invalid;
+    }
+
+    return BlockValidationResult::Missing;
+}
+
 std::set<int64_t> AudacityProject::validateBlocks() const
 {
     std::set<int64_t> missingBlocks;
@@ -257,51 +336,106 @@ std::set<int64_t> AudacityProject::validateBlocks() const
         if (missingBlocks.count(block.getBlockId()))
             continue;
 
-        try
-        {
-            // Creating query every time is slower,
-            // but can withstand error
-            SQLite::Statement query(
-                mDb.DB(),
-                "SELECT sampleformat, summin, summax, sumrms, summary256, summary64k, samples FROM sampleblocks WHERE blockid = ?1;");
-
-            query.bind(1, block.getBlockId());
-
-            int rowsCount = 0;
-
-            while (query.executeStep())
-            {
-                const int format = query.getColumn(0).getInt();
-
-                if (block.getParent()->getFormat() != format)
-                    throw std::runtime_error(fmt::format("Format mismatch for block {}\n", block.getBlockId()));
-
-                ++rowsCount;
-            }
-
-            if (rowsCount == 0)
-                throw std::runtime_error("Block not found");
-        }
-        catch (const std::exception& ex)
+        if (auto result = validateBlock(block); result != BlockValidationResult::Ok)
         {
             missingBlocks.emplace(block.getBlockId());
-            fmt::print("Invalid block {}: {}\n", block.getBlockId(), ex.what());
-        }
 
+            if (result == BlockValidationResult::Invalid)
+                fmt::print("Wrong sample formate for block: {}\n", block.getBlockId());
+            else if (result == BlockValidationResult::Missing)
+                fmt::print("Missing block: {}\n", block.getBlockId());
+        }
     }
 
     return missingBlocks;
 }
 
-std::set<int64_t> AudacityProject::fixupMissingBlocks()
+std::set<int64_t> AudacityProject::recoverProject()
 {
     auto missingBlocks = validateBlocks();
 
-    for (auto& block : mWaveBlocks)
+    for (auto& sequence : mSequences)
     {
+        int nextBlockStart = 0;
+
+        bool firstBlock = true;
+
+        for (auto blockPtr : sequence)
+        {
+            if (firstBlock)
+            {
+                firstBlock = false;
+
+                if (blockPtr->getStart() != 0)
+                {
+                    fmt::print("Invalid block id for first block in sequence: {}\n", blockPtr->getBlockId());
+                    blockPtr->setBlockId(0);
+                }
+
+                nextBlockStart += missingBlocks.count(blockPtr->getBlockId()) ?
+                                      blockPtr->getLength() :
+                                      getRealBlockLength(*blockPtr);
+
+                continue;
+            }
+
+            if (missingBlocks.count(blockPtr->getBlockId()))
+            {
+                // We can only hope that the block start is correct
+                nextBlockStart += blockPtr->getLength();
+            }
+            else
+            {
+                if (blockPtr->getStart() != nextBlockStart)
+                {
+                    fmt::print("Invalid block start for block: {}\n", blockPtr->getBlockId());
+                    blockPtr->setStart(nextBlockStart);
+                }
+
+                nextBlockStart += getRealBlockLength(*blockPtr);
+            }
+        }
+    }
+
+    const auto blocksCount = mWaveBlocks.size();
+
+    for (size_t index = 0; index < blocksCount; ++index)
+    {
+        auto& block = mWaveBlocks[index];
+
         if (missingBlocks.count(block.getBlockId()) == 0)
             continue;
 
+        if (index > 0 && (index + 1) < blocksCount)
+        {
+            auto& prevBlock = mWaveBlocks[index - 1];
+            auto& nextBlock = mWaveBlocks[index + 1];
+
+            if (prevBlock.getParent () == block.getParent () &&
+                nextBlock.getParent () == block.getParent ())
+            {
+                const auto prevBlockId = prevBlock.getBlockId();
+                const auto nextBlockId = nextBlock.getBlockId();
+
+                const auto diff = nextBlockId - prevBlockId;
+
+                if (prevBlockId > 0 && nextBlockId > 0 &&
+                    (diff == 2 || diff == 4))
+                {
+                    const auto potentialBlockId =
+                        (prevBlockId + nextBlockId) / 2;
+
+                    if (containsBlock(potentialBlockId))
+                    {
+                        fmt::print("Recovered block id: {} -> {}\n", block.getBlockId(), potentialBlockId);
+                        block.setBlockId(potentialBlockId);
+                        continue;
+                    }
+                }
+            }
+        }
+
+        fmt::print("Converting block to silence: start {}\n", block.getStart());
         block.convertToSilence();
     }
 
@@ -349,15 +483,22 @@ void AudacityProject::removeUnusedBlocks()
 
     readBlocksList.reset();
 
-    std::set<int64_t> orphanedBlocks;
+    std::set<int64_t> usedBlocks;
 
-    for (const auto block : mWaveBlocks)
+    for (const auto& block : mWaveBlocks)
     {
         if (block.isSilence())
             continue;
 
-        if (availableBlocks.count(block.getBlockId()) == 0)
-            orphanedBlocks.emplace(block.getBlockId());
+        usedBlocks.emplace(block.getBlockId());
+    }
+
+    std::set<int64_t> orphanedBlocks;
+
+    for (auto blockId : availableBlocks)
+    {
+        if (usedBlocks.count(blockId) == 0)
+            orphanedBlocks.emplace(blockId);
     }
 
     mDb.reopenReadonlyAsWritable();
@@ -470,6 +611,109 @@ void AudacityProject::extractClips() const
     }
 }
 
+namespace
+{
+std::string FormatTime(double seconds)
+{
+    if (seconds < 0)
+        return fmt::format("{}", seconds);
+
+    if (seconds > 60 * 60)
+        return fmt::format(
+            "{:02}:{:02}:{:02}.{:03}", (int)seconds / 3600, (int)seconds / 60 % 60,
+            (int)seconds % 60, (int)(seconds * 1000) % 1000);
+    else if (seconds > 60)
+        return fmt::format(
+            "{:02}:{:02}.{:03}", (int)seconds / 60, (int)seconds % 60,
+            (int)(seconds * 1000) % 1000);
+    else
+        return fmt::format(
+            "{:02}.{:03}", (int)seconds, (int)(seconds * 1000) % 1000);
+}
+
+struct BlockStatistics final
+{
+    size_t totalUsageCount {};
+    size_t audibleUsageCount {};
+};
+}
+
+void AudacityProject::printProjectStatistics() const
+{
+    std::unordered_map<int64_t, BlockStatistics> blocksStatistics;
+
+    for (const auto& track : mWaveTracks)
+    {
+        fmt::print(
+            "Track {}: {}\n", track.getParentIndex(), track.getTrackName());
+
+        for (auto clip : track.getClips())
+        {
+            const int64_t firstSample =
+                int64_t((clip->getTrimLeft()) * track.getSampleRate());
+
+            const int64_t lastSampleOffset =
+                int64_t(clip->getTrimRight() * track.getSampleRate());
+
+            size_t numSamples = 0;
+
+            for (auto sequence : *clip)
+            {
+                numSamples += sequence->getNumSamples();
+
+                const int64_t lastSample = sequence->getNumSamples() - lastSampleOffset;
+
+                for (const auto& block : *sequence)
+                {
+                    auto& blockStats = blocksStatistics[block->getBlockId()];
+
+                    ++blockStats.totalUsageCount;
+
+                    if ((block->getStart() + block->getLength()) >= firstSample &&
+                        block->getStart() < lastSample)
+                        ++blockStats.audibleUsageCount;
+                }
+            }
+
+            const double totalClipTime =
+                double(numSamples) / track.getSampleRate();
+
+            const double trimmedClipTime =
+                totalClipTime - clip->getTrimLeft() - clip->getTrimRight();
+            
+            fmt::print(
+                "\tClip {}: '{}'.\n\t\tTotal samples {}\n\t\tTotal time: {}\n\t\tTrimmed time: {}\n\t\tTrimmed / Total: {:.4f}%\n",
+                clip->getParentIndex(), clip->getName(), numSamples,
+                FormatTime(totalClipTime), FormatTime(trimmedClipTime),
+                trimmedClipTime / totalClipTime * 100.0);
+        }
+    }
+
+    const auto silentBlocksCount = std::count_if(
+        blocksStatistics.begin(), blocksStatistics.end(),
+        [](const auto& p) { return p.second.audibleUsageCount == 0; });
+
+    const auto unsharedBlocksCount = std::count_if(
+        blocksStatistics.begin(), blocksStatistics.end(),
+        [](const auto& p) { return p.second.totalUsageCount == 1; });
+
+    const auto unsharedSilentBlocks = std::count_if(
+        blocksStatistics.begin(), blocksStatistics.end(),
+        [](const auto& p) {
+            return p.second.audibleUsageCount == 0 &&
+                   p.second.totalUsageCount == 1;
+        });
+
+    fmt::print(
+        "Total blocks in project: {}\n\tSilent blocks count: {} ({:02.5}%)\nNot shared blocks count: {} ({:02.5}%)\n\tSilent blocks count: {} ({:02.5}%)\n",
+        blocksStatistics.size(), silentBlocksCount,
+        double(silentBlocksCount) / blocksStatistics.size() * 100.0,
+        unsharedBlocksCount,
+        double(unsharedBlocksCount) / blocksStatistics.size() * 100.0,
+        unsharedSilentBlocks,
+        double(unsharedSilentBlocks) / unsharedBlocksCount * 100.0);
+}
+
 std::string_view AudacityProject::CacheString(std::string_view view, bool reuse)
 {
     if (reuse)

src/ProjectModel.h

@@ -54,6 +54,9 @@ public:
     bool isSilence() const noexcept;
     void convertToSilence() noexcept;
 
+    void setBlockId(int64_t blockId) noexcept;
+    void setStart(int64_t start) noexcept;
+
     int64_t getBlockId() const noexcept;
     int64_t getStart() const noexcept;
     int64_t getLength() const noexcept;
@@ -144,6 +147,8 @@ public:
     int getSampleFormat() const;
     int getSampleRate() const;
 
+    const std::vector<Clip*>& getClips() const;
+
 private:
     size_t mParentIndex;
 
@@ -165,15 +170,31 @@ public:
     AudacityProject(AudacityDatabase& db);
     ~AudacityProject();
 
+    bool containsBlock(int64_t blockId) const;
+
+    enum class BlockValidationResult
+    {
+        Ok,
+        Missing,
+        Invalid
+    };
+
+    int getRealBlockLength(const WaveBlock& block) const;
+
+    BlockValidationResult validateBlock(const WaveBlock& block) const;
+
     std::set<int64_t> validateBlocks() const;
 
-    std::set<int64_t> fixupMissingBlocks();
+    std::set<int64_t> recoverProject();
 
     void saveProject();
 
     void removeUnusedBlocks();
 
     void extractClips() const;
+
+    void printProjectStatistics() const;
+
 private:
     std::string_view CacheString(std::string_view view, bool reuse);
 

src/SampleFormat.cpp

@@ -34,3 +34,18 @@ uint32_t BytesPerSample(SampleFormat format)
         throw std::runtime_error(fmt::format("Unsupported format {}", format));
     }
 }
+
+uint32_t DiskBytesPerSample(SampleFormat format)
+{
+    switch (format)
+    {
+    case SampleFormat::Int16:
+        return 2;
+    case SampleFormat::Int24:
+        return 4;
+    case SampleFormat::Float32:
+        return 4;
+    default:
+        throw std::runtime_error(fmt::format("Unsupported format {}", format));
+    }
+}

src/SampleFormat.h

@@ -18,3 +18,4 @@ enum class SampleFormat
 SampleFormat SampleFormatFromString(std::string_view format);
 
 uint32_t BytesPerSample(SampleFormat format);
+uint32_t DiskBytesPerSample(SampleFormat format);

src/XMLHandler.h

@@ -55,7 +55,15 @@ void GetAttributeValue(const AttributeValue& attr, Ret& result)
             {
                 if constexpr (std::is_same_v<Ret, bool>)
                 {
-                    result = arg == "true" || arg == "0";
+                    result = arg == "true" || arg == "1";
+                }
+                else if constexpr (std::is_floating_point_v<Ret>)
+                {
+                    std::string tmp(arg);
+                    if constexpr (std::is_same_v<Ret, float>)
+                        result = std::stof(tmp);
+                    else
+                        result = static_cast<Ret>(std::stod(tmp));
                 }
                 else
                 {

src/main.cpp

@@ -27,6 +27,7 @@
 DEFINE_bool(drop_autosave, false, "Drop autosave table, if exists");
 DEFINE_bool(extract_project, false, "Extract Audacity project as an XML file");
 DEFINE_bool(check_integrity, false, "Check AUP3 integrity");
+DEFINE_bool(analyze_project, false, "Print project statistics");
 
 DEFINE_bool(compact, false, "Compact the project");
 
@@ -49,8 +50,6 @@ DEFINE_string(
     "float",
     "Sample format for the extracted samples (-extract_sample_blocks, -extract_as_mono_track, -extract_as_stereo_track). Possible values are: int16, int24, float");
 
-constexpr int64_t AudacityProjectID = 1096107097;
-constexpr int64_t MaxSupportedVersion = (3 << 24) + (1 << 16) + (3 << 8) + 0; // 3.1.3.0
 
 namespace
 {
@@ -182,7 +181,7 @@ int main(int argc, char **argv)
             if (project == nullptr)
                 project = std::make_unique<AudacityProject>(projectDatabase);
 
-            project->fixupMissingBlocks();
+            project->recoverProject();
         }
 
         if (FLAGS_compact)
@@ -193,6 +192,14 @@ int main(int argc, char **argv)
             project->removeUnusedBlocks();
         }
 
+        if (FLAGS_analyze_project)
+        {
+            if (project == nullptr)
+                project = std::make_unique<AudacityProject>(projectDatabase);
+
+            project->printProjectStatistics();
+        }
+
         if (FLAGS_extract_clips)
         {
             if (project == nullptr)
@@ -219,6 +226,11 @@ int main(int argc, char **argv)
                 SampleFormatFromString(FLAGS_sample_format), FLAGS_sample_rate, true);
         }
     }
+    catch (const fmt::format_error& err)
+    {
+        fmt::print("{}\n", err.what());
+        return -1;
+    }
     catch (const std::exception& ex)
     {
         fmt::print("{}\n", ex.what());