Change the way Conan is collected

1. Tool ensures that the waveblock `start`is correct whenever possible.
2. Tool tries to guess the block id from the sequence

.github/workflows/cmake.yml

@@ -38,7 +38,7 @@ jobs:
         #cache-dependency-path: # optional
     - uses: BSFishy/pip-action@v1
       with:
-        packages: conan
+        packages: conan<2
     - name: Cache for .conan
       id: cache-conan
       uses: actions/cache@v2

.gitignore

@@ -34,3 +34,4 @@
 build/**
 .vs/**
 out/**
+.venv/**

CMakeLists.txt

@@ -166,7 +166,7 @@ set(CPACK_PACKAGE_NAME "audacity-project-tools")
 set(CPACK_PACKAGE_VENDOR "Audacity")
 set(CPACK_PACKAGE_VERSION_MAJOR 1)
 set(CPACK_PACKAGE_VERSION_MINOR 0)
-set(CPACK_PACKAGE_VERSION_PATCH 2)
+set(CPACK_PACKAGE_VERSION_PATCH 4)
 set(CPACK_GENERATOR ZIP)
 SET(CPACK_OUTPUT_FILE_PREFIX packages)
 

README.md

@@ -48,4 +48,4 @@ $ ls
 
 `audacity-project-tools` requires a C++17 compliant compiler with the complete C++17 library. The build requires <filesystem> and floating-point versions of `from_chars`.
 
-CMake and Conan are required to configure the project.
+CMake and Conan are required to configure the project. Conan 2.0 is not supported ATM.

requirements.txt

@@ -1,21 +1 @@
-bottle==0.12.19
-certifi==2021.10.8
-charset-normalizer==2.0.9
-colorama==0.4.4
-conan==1.43.2
-distro==1.6.0
-fasteners==0.16.3
-idna==3.3
-Jinja2==2.11.3
-MarkupSafe==2.0.1
-node-semver==0.6.1
-patch-ng==1.17.4
-pluginbase==1.0.1
-Pygments==2.10.0
-PyJWT==1.7.1
-python-dateutil==2.8.2
-PyYAML==5.4.1
-requests==2.26.0
-six==1.16.0
-tqdm==4.62.3
-urllib3==1.26.7
+conan==1.59.0

src/AudacityDatabase.cpp

@@ -25,7 +25,7 @@ namespace
 
 constexpr int64_t AudacityProjectID = 1096107097;
 constexpr int64_t MaxSupportedVersion =
-    (3 << 24) + (1 << 16) + (3 << 8) + 0; // 3.1.3.0
+    (3 << 24) + (4 << 16) + (0 << 8) + 0; // 3.4.0.0
 
 template<typename T> size_t readInt(const std::string& string, size_t offset, T& output)
 {
@@ -112,13 +112,23 @@ AudacityDatabase::AudacityDatabase(
             mProjectVersion =
                 mDatabase->execAndGet("PRAGMA user_version;").getUInt();
 
+            const auto majorVersion = (mProjectVersion >> 24) & 0xFF;
+            const auto minorVersion = (mProjectVersion >> 16) & 0xFF;
+            const auto patchVersion = (mProjectVersion >> 8) & 0xFF;
+
             fmt::print(
-                "Project requires Audacity {}.{}.{}\n",
-                (mProjectVersion >> 24) & 0xFF, (mProjectVersion >> 16) & 0xFF,
-                (mProjectVersion >> 8) & 0xFF);
+                "Project requires Audacity {}.{}.{}\n", majorVersion,
+                minorVersion, patchVersion);
+
+            if (majorVersion != 3)
+            {
+                fmt::print("user_version pragma appears to be broken...\n");
+                mProjectVersion = MaxSupportedVersion;
+            }
 
             if (mProjectVersion > MaxSupportedVersion)
-                throw std::runtime_error("Unsupported project version");
+                fmt::print(
+                    "DANGER!!! Unsupported Audacity version detected! Project data might be lost! Proceed with caution!\n");
         }, true);
 }
 

src/ProjectModel.cpp

@@ -48,6 +48,20 @@ void WaveBlock::convertToSilence() noexcept
     mXMLNode->setAttribute("badblock", true);
 }
 
+void WaveBlock::setBlockId(int64_t blockId) noexcept
+{
+    mBlockId = blockId;
+    mXMLNode->setAttribute("blockid", mBlockId);
+    mXMLNode->setAttribute("badblock", true);
+}
+
+void WaveBlock::setStart(int64_t start) noexcept
+{
+    mStart = start;
+    mXMLNode->setAttribute("start", mStart);
+    mXMLNode->setAttribute("badblock", true);
+}
+
 int64_t WaveBlock::getBlockId() const noexcept
 {
     return mBlockId;
@@ -250,6 +264,66 @@ AudacityProject::~AudacityProject()
 {
 }
 
+bool AudacityProject::containsBlock(int64_t blockId) const
+{
+    SQLite::Statement query(
+        mDb.DB(), "SELECT 1 FROM sampleblocks WHERE blockid = ?1;");
+
+    query.bind(1, blockId);
+
+    while (query.executeStep())
+        return true;
+
+    return false;
+}
+
+int AudacityProject::getRealBlockLength(const WaveBlock& block) const
+{
+    if (block.getBlockId() < 0)
+        return -block.getBlockId();
+
+    SQLite::Statement query (
+        mDb.DB(), "SELECT sampleformat, samples FROM sampleblocks WHERE blockid = ?1;");
+
+    query.bind(1, block.getBlockId());
+
+    while (query.executeStep ())
+    {
+        const int format = query.getColumn(0).getInt();
+        const int64_t blobSize = query.getColumn(1).getBytes();
+
+        if (format != block.getParent()->getFormat())
+            throw std::runtime_error (
+                fmt::format("Unexpected sample format for block {}",
+                                   block.getBlockId()));
+
+        return blobSize / BytesPerSample(static_cast<SampleFormat>(format));
+    }
+
+    throw std::runtime_error (
+        fmt::format("Block {} not found in the database", block.getBlockId()));
+}
+
+AudacityProject::BlockValidationResult
+AudacityProject::validateBlock(const WaveBlock& block) const
+{
+    // Creating query every time is slower,
+    // but can withstand error
+    SQLite::Statement query(
+        mDb.DB(), "SELECT sampleformat FROM sampleblocks WHERE blockid = ?1;");
+
+    query.bind(1, block.getBlockId());
+
+    while (query.executeStep())
+    {
+        const int format = query.getColumn(0).getInt();
+
+        return block.getParent()->getFormat() == format ? BlockValidationResult::Ok : BlockValidationResult::Invalid;
+    }
+
+    return BlockValidationResult::Missing;
+}
+
 std::set<int64_t> AudacityProject::validateBlocks() const
 {
     std::set<int64_t> missingBlocks;
@@ -262,51 +336,106 @@ std::set<int64_t> AudacityProject::validateBlocks() const
         if (missingBlocks.count(block.getBlockId()))
             continue;
 
-        try
-        {
-            // Creating query every time is slower,
-            // but can withstand error
-            SQLite::Statement query(
-                mDb.DB(),
-                "SELECT sampleformat, summin, summax, sumrms, summary256, summary64k, samples FROM sampleblocks WHERE blockid = ?1;");
-
-            query.bind(1, block.getBlockId());
-
-            int rowsCount = 0;
-
-            while (query.executeStep())
-            {
-                const int format = query.getColumn(0).getInt();
-
-                if (block.getParent()->getFormat() != format)
-                    throw std::runtime_error(fmt::format("Format mismatch for block {}\n", block.getBlockId()));
-
-                ++rowsCount;
-            }
-
-            if (rowsCount == 0)
-                throw std::runtime_error("Block not found");
-        }
-        catch (const std::exception& ex)
+        if (auto result = validateBlock(block); result != BlockValidationResult::Ok)
         {
             missingBlocks.emplace(block.getBlockId());
-            fmt::print("Invalid block {}: {}\n", block.getBlockId(), ex.what());
-        }
 
+            if (result == BlockValidationResult::Invalid)
+                fmt::print("Wrong sample formate for block: {}\n", block.getBlockId());
+            else if (result == BlockValidationResult::Missing)
+                fmt::print("Missing block: {}\n", block.getBlockId());
+        }
     }
 
     return missingBlocks;
 }
 
-std::set<int64_t> AudacityProject::fixupMissingBlocks()
+std::set<int64_t> AudacityProject::recoverProject()
 {
     auto missingBlocks = validateBlocks();
 
-    for (auto& block : mWaveBlocks)
+    for (auto& sequence : mSequences)
     {
+        int nextBlockStart = 0;
+
+        bool firstBlock = true;
+
+        for (auto blockPtr : sequence)
+        {
+            if (firstBlock)
+            {
+                firstBlock = false;
+
+                if (blockPtr->getStart() != 0)
+                {
+                    fmt::print("Invalid block id for first block in sequence: {}\n", blockPtr->getBlockId());
+                    blockPtr->setBlockId(0);
+                }
+
+                nextBlockStart += missingBlocks.count(blockPtr->getBlockId()) ?
+                                      blockPtr->getLength() :
+                                      getRealBlockLength(*blockPtr);
+
+                continue;
+            }
+
+            if (missingBlocks.count(blockPtr->getBlockId()))
+            {
+                // We can only hope that the block start is correct
+                nextBlockStart += blockPtr->getLength();
+            }
+            else
+            {
+                if (blockPtr->getStart() != nextBlockStart)
+                {
+                    fmt::print("Invalid block start for block: {}\n", blockPtr->getBlockId());
+                    blockPtr->setStart(nextBlockStart);
+                }
+
+                nextBlockStart += getRealBlockLength(*blockPtr);
+            }
+        }
+    }
+
+    const auto blocksCount = mWaveBlocks.size();
+
+    for (size_t index = 0; index < blocksCount; ++index)
+    {
+        auto& block = mWaveBlocks[index];
+
         if (missingBlocks.count(block.getBlockId()) == 0)
             continue;
 
+        if (index > 0 && (index + 1) < blocksCount)
+        {
+            auto& prevBlock = mWaveBlocks[index - 1];
+            auto& nextBlock = mWaveBlocks[index + 1];
+
+            if (prevBlock.getParent () == block.getParent () &&
+                nextBlock.getParent () == block.getParent ())
+            {
+                const auto prevBlockId = prevBlock.getBlockId();
+                const auto nextBlockId = nextBlock.getBlockId();
+
+                const auto diff = nextBlockId - prevBlockId;
+
+                if (prevBlockId > 0 && nextBlockId > 0 &&
+                    (diff == 2 || diff == 4))
+                {
+                    const auto potentialBlockId =
+                        (prevBlockId + nextBlockId) / 2;
+
+                    if (containsBlock(potentialBlockId))
+                    {
+                        fmt::print("Recovered block id: {} -> {}\n", block.getBlockId(), potentialBlockId);
+                        block.setBlockId(potentialBlockId);
+                        continue;
+                    }
+                }
+            }
+        }
+
+        fmt::print("Converting block to silence: start {}\n", block.getStart());
         block.convertToSilence();
     }
 

src/ProjectModel.h

@@ -54,6 +54,9 @@ public:
     bool isSilence() const noexcept;
     void convertToSilence() noexcept;
 
+    void setBlockId(int64_t blockId) noexcept;
+    void setStart(int64_t start) noexcept;
+
     int64_t getBlockId() const noexcept;
     int64_t getStart() const noexcept;
     int64_t getLength() const noexcept;
@@ -167,9 +170,22 @@ public:
     AudacityProject(AudacityDatabase& db);
     ~AudacityProject();
 
+    bool containsBlock(int64_t blockId) const;
+
+    enum class BlockValidationResult
+    {
+        Ok,
+        Missing,
+        Invalid
+    };
+
+    int getRealBlockLength(const WaveBlock& block) const;
+
+    BlockValidationResult validateBlock(const WaveBlock& block) const;
+
     std::set<int64_t> validateBlocks() const;
 
-    std::set<int64_t> fixupMissingBlocks();
+    std::set<int64_t> recoverProject();
 
     void saveProject();
 

src/SampleFormat.cpp

@@ -34,3 +34,18 @@ uint32_t BytesPerSample(SampleFormat format)
         throw std::runtime_error(fmt::format("Unsupported format {}", format));
     }
 }
+
+uint32_t DiskBytesPerSample(SampleFormat format)
+{
+    switch (format)
+    {
+    case SampleFormat::Int16:
+        return 2;
+    case SampleFormat::Int24:
+        return 4;
+    case SampleFormat::Float32:
+        return 4;
+    default:
+        throw std::runtime_error(fmt::format("Unsupported format {}", format));
+    }
+}

src/SampleFormat.h

@@ -18,3 +18,4 @@ enum class SampleFormat
 SampleFormat SampleFormatFromString(std::string_view format);
 
 uint32_t BytesPerSample(SampleFormat format);
+uint32_t DiskBytesPerSample(SampleFormat format);

src/main.cpp

@@ -50,8 +50,6 @@ DEFINE_string(
     "float",
     "Sample format for the extracted samples (-extract_sample_blocks, -extract_as_mono_track, -extract_as_stereo_track). Possible values are: int16, int24, float");
 
-constexpr int64_t AudacityProjectID = 1096107097;
-constexpr int64_t MaxSupportedVersion = (3 << 24) + (1 << 16) + (3 << 8) + 0; // 3.1.3.0
 
 namespace
 {
@@ -183,7 +181,7 @@ int main(int argc, char **argv)
             if (project == nullptr)
                 project = std::make_unique<AudacityProject>(projectDatabase);
 
-            project->fixupMissingBlocks();
+            project->recoverProject();
         }
 
         if (FLAGS_compact)