fix(render): follow-ups to upscaled textures: canvas pixels and size limit

src/source/Engine/Object/ZzzInventory.cpp

@@ -7262,6 +7262,12 @@ void CreateCastleMark(int Type, BYTE* buffer, bool blend)
     Width = (int)b->Width;
     Height = (int)b->Height;
     BYTE* Buffer = b->Buffer;
+    if (!Buffer || Width == 0 || Height == 0)
+    {
+        mu::log::Get("gameplay")
+            ->error("[CreateCastleMark] bitmap {} not loaded (BitmapIndex={})", Type, b->BitmapIndex);
+        return;
+    }
 
     int alpha = 128;
     if (blend)

src/source/Engine/Object/ZzzOpenData.cpp

@@ -3498,7 +3498,7 @@ void OpenImages()
     LoadBitmap(L"Interface\\lock_02.jpg", BITMAP_INVENTORY_BUTTON + 15);
     LoadBitmap(L"Interface\\lock_03.jpg", BITMAP_INVENTORY_BUTTON + 16);
     LoadBitmap(L"Interface\\lock_04.jpg", BITMAP_INVENTORY_BUTTON + 17);
-    LoadBitmap(L"Interface\\guild.tga", BITMAP_GUILD);
+    LoadCanvasBitmap(L"Interface\\guild.tga", BITMAP_GUILD);
 }
 
 void OpenSounds()
@@ -4197,7 +4197,7 @@ void OpenBasicData(HDC hDC)
     LoadBitmap(L"Interface\\DoorCL.jpg", BITMAP_INTERFACE_EX + 40);
     LoadBitmap(L"Interface\\DoorOP.jpg", BITMAP_INTERFACE_EX + 41);
     //OpenJpeg( "Effect\\FireSnuff.jpg"       , BITMAP_FIRE_SNUFF,    GL_LINEAR, GL_CLAMP_TO_EDGE );
-    LoadBitmap(L"Object31\\Flag.tga", BITMAP_INTERFACE_MAP + 0, GL_LINEAR, GL_CLAMP_TO_EDGE);
+    LoadCanvasBitmap(L"Object31\\Flag.tga", BITMAP_INTERFACE_MAP + 0, GL_LINEAR, GL_CLAMP_TO_EDGE);
     LoadBitmap(L"Interface\\CursorAttack2.tga", BITMAP_CURSOR2);
     LoadBitmap(L"Effect\\Cratered.tga", BITMAP_CRATER);
     LoadBitmap(L"Effect\\FormationMark.tga", BITMAP_FORMATION_MARK);

src/source/Render/Sprites/GlobalBitmap.cpp

@@ -184,17 +184,33 @@ SDL_GPUSampler* CreateSampler(SDL_GPUDevice* device, SDL_GPUFilter filter, SDL_G
     return SDL_CreateGPUSampler(device, &samplerInfo);
 }
 
-// The CPU copy of an upscaled texture is up to 16 times the original and
-// nothing reads it after the upload: only the canvases (BITMAP_FONT,
-// BITMAP_GUILD) are redrawn through Buffer, and those never carry the marker.
-// Keeping the copies took the 32 bit client past 3.3 GB of address space.
-// Returns the bytes released.
-std::size_t ReleaseUploadedPixels(BITMAP_t& bitmap)
+// Nothing reads the CPU copy of a texture after the upload except for the
+// canvases (LoadCanvasBitmap), which are redrawn through Buffer. The copy of an
+// upscaled texture is up to 16 times the original; keeping the copies took the
+// 32 bit client past 3.3 GB of address space.
+void ReleaseUploadedPixels(BITMAP_t& bitmap)
 {
-    const std::size_t released = bitmap.BufferStorage.size();
     std::vector<std::uint8_t>().swap(bitmap.BufferStorage);
     bitmap.Buffer = nullptr;
-    return released;
+}
+
+// The size of the base level of an uploaded texture, which the loader counts as
+// texture memory.
+std::uint32_t UploadedBytes(int width, int height)
+{
+    return static_cast<std::uint32_t>(width) * static_cast<std::uint32_t>(height) * 4u;
+}
+
+// A canvas is drawn on through Buffer at the original size, so an upscaled file
+// loaded into one is brought back to that size. Width and height become the
+// size to upload.
+void ShrinkCanvas(BITMAP_t& bitmap, int& width, int& height, int scale)
+{
+    bitmap.BufferStorage =
+        Render::Textures::Upscale::ShrinkToOriginal(bitmap.BufferStorage, width, height, bitmap.Components, scale);
+    bitmap.Buffer = bitmap.BufferStorage.data();
+    width /= scale;
+    height /= scale;
 }
 
 bool UploadTextureSDLGpu(BITMAP_t* bitmap, const std::uint8_t* pixelData, int width, int height, SDL_GPUFilter filter,
@@ -609,7 +625,8 @@ GLuint CGlobalBitmap::LoadImageExclusive(const std::wstring& filename, GLuint ui
     }
     return BITMAP_UNKNOWN;
 }
-bool CGlobalBitmap::LoadImage(GLuint uiBitmapIndex, const std::wstring& filename, GLuint uiFilter, GLuint uiWrapMode, bool padToPowerOfTwo, bool premultiplyAlpha)
+bool CGlobalBitmap::LoadImage(GLuint uiBitmapIndex, const std::wstring& filename, GLuint uiFilter, GLuint uiWrapMode,
+                              bool padToPowerOfTwo, bool premultiplyAlpha, bool keepPixels)
 {
     unsigned int UICLAMP = GL_CLAMP_TO_EDGE;
     unsigned int UIREPEAT = GL_REPEAT;
@@ -657,9 +674,9 @@ bool CGlobalBitmap::LoadImage(GLuint uiBitmapIndex, const std::wstring& filename
 
     bool loaded = false;
     if (0 == _wcsicmp(ext.c_str(), L"jpg"))
-        loaded = OpenJpegTurbo(uiBitmapIndex, filename, uiFilter, uiWrapMode, padToPowerOfTwo);
+        loaded = OpenJpegTurbo(uiBitmapIndex, filename, uiFilter, uiWrapMode, padToPowerOfTwo, keepPixels);
     else if (0 == _wcsicmp(ext.c_str(), L"tga"))
-        loaded = OpenTga(uiBitmapIndex, filename, uiFilter, uiWrapMode, padToPowerOfTwo, premultiplyAlpha);
+        loaded = OpenTga(uiBitmapIndex, filename, uiFilter, uiWrapMode, padToPowerOfTwo, premultiplyAlpha, keepPixels);
 #ifdef _EDITOR
     if (loaded)
         m_loadWorlds[uiBitmapIndex] = Core::AssetLoadWorld::Get();
@@ -692,8 +709,7 @@ void CGlobalBitmap::UnloadImage(GLuint uiBitmapIndex, bool bForce)
                 }
             }
 
-            const auto memoryUsed = static_cast<std::uint32_t>(pBitmap->BufferStorage.size());
-            m_dwUsedTextureMemory -= memoryUsed;
+            m_dwUsedTextureMemory -= pBitmap->UploadedBytes;
 
             m_mapBitmap.erase(mi);
 #ifdef _EDITOR
@@ -879,7 +895,7 @@ GLuint CGlobalBitmap::FindAvailableTextureIndex(GLuint uiSeed)
 }
 
 bool CGlobalBitmap::OpenJpegTurbo(GLuint uiBitmapIndex, const std::wstring& filename, GLuint uiFilter,
-                                  GLuint uiWrapMode, bool padToPowerOfTwo)
+                                  GLuint uiWrapMode, bool padToPowerOfTwo, bool keepPixels)
 {
     std::wstring filename_ozj;
     ExchangeExt(filename, L"OZJ", filename_ozj);
@@ -902,6 +918,11 @@ bool CGlobalBitmap::OpenJpegTurbo(GLuint uiBitmapIndex, const std::wstring& file
         return false;
     }
 
+    // A marked file holds an image up to its scale times the largest original.
+    const int markedScale = Render::Textures::Upscale::ReadMarkedScale(jpegBuf);
+    const int maxWidth = MAX_WIDTH * markedScale;
+    const int maxHeight = MAX_HEIGHT * markedScale;
+
     // Skip first 24 bytes (OZJ header)
     const unsigned char* jpegData = jpegBuf.data() + 24;
     const auto jpegSize = static_cast<unsigned long>(jpegBuf.size() - 24);
@@ -919,11 +940,17 @@ bool CGlobalBitmap::OpenJpegTurbo(GLuint uiBitmapIndex, const std::wstring& file
 
     auto headerResult =
         tjDecompressHeader3(tjHandle.get(), jpegData, jpegSize, &jpegWidth, &jpegHeight, &jpegSubsamp, &jpegColorspace);
-    if (headerResult != 0 || jpegWidth <= 0 || jpegHeight <= 0 || jpegWidth > MAX_WIDTH || jpegHeight > MAX_HEIGHT)
+    if (headerResult != 0 || jpegWidth <= 0 || jpegHeight <= 0)
     {
         ReportTurboError(L"tjDecompressHeader3");
         return false;
     }
+    if (jpegWidth > maxWidth || jpegHeight > maxHeight)
+    {
+        g_ErrorReport.Write(L"OpenJpegTurbo: image too large %ls (%dx%d, at most %dx%d)\r\n", filename_ozj.c_str(),
+                            jpegWidth, jpegHeight, maxWidth, maxHeight);
+        return false;
+    }
 
     std::vector<unsigned char> decompressedBuffer(static_cast<std::size_t>(jpegWidth) *
                                                   static_cast<std::size_t>(jpegHeight) * 3u);
@@ -935,14 +962,14 @@ bool CGlobalBitmap::OpenJpegTurbo(GLuint uiBitmapIndex, const std::wstring& file
         return false;
     }
 
-    const int textureWidth = padToPowerOfTwo ? NextPowerOfTwo(jpegWidth, MAX_WIDTH) : jpegWidth;
-    const int textureHeight = padToPowerOfTwo ? NextPowerOfTwo(jpegHeight, MAX_HEIGHT) : jpegHeight;
+    const int textureWidth = padToPowerOfTwo ? NextPowerOfTwo(jpegWidth, maxWidth) : jpegWidth;
+    const int textureHeight = padToPowerOfTwo ? NextPowerOfTwo(jpegHeight, maxHeight) : jpegHeight;
     const int scale = Render::Textures::Upscale::ReadScale(jpegBuf, textureWidth, textureHeight);
+    const bool upscaled = scale > Render::Textures::Upscale::NoScale && !keepPixels;
 
     auto pNewBitmap = CreateBitmapStorage(uiBitmapIndex, filename, textureWidth, textureHeight, 3);
     pNewBitmap->Width = static_cast<float>(textureWidth / scale);
     pNewBitmap->Height = static_cast<float>(textureHeight / scale);
-    m_dwUsedTextureMemory += static_cast<std::uint32_t>(pNewBitmap->BufferStorage.size());
 
     const int jpegRowSize = jpegWidth * 3;
     const int textureRowSize = textureWidth * 3;
@@ -964,17 +991,26 @@ bool CGlobalBitmap::OpenJpegTurbo(GLuint uiBitmapIndex, const std::wstring& file
                static_cast<std::size_t>(jpegHeight) * static_cast<std::size_t>(jpegWidth) * 3u);
     }
 
+    int uploadWidth = textureWidth;
+    int uploadHeight = textureHeight;
+    if (keepPixels && scale > Render::Textures::Upscale::NoScale)
+    {
+        ShrinkCanvas(*pNewBitmap, uploadWidth, uploadHeight, scale);
+    }
+
     pNewBitmap->TextureNumber = uiBitmapIndex;
-    std::vector<std::uint8_t> rgbaData = PadRGBToRGBA(pNewBitmap->Buffer, textureWidth, textureHeight);
-    if (!UploadTextureSDLGpu(pNewBitmap.get(), rgbaData.data(), textureWidth, textureHeight, MapGLFilterToSDL(uiFilter),
-                             MapGLWrapToSDL(uiWrapMode), scale > Render::Textures::Upscale::NoScale))
+    std::vector<std::uint8_t> rgbaData = PadRGBToRGBA(pNewBitmap->Buffer, uploadWidth, uploadHeight);
+    if (!UploadTextureSDLGpu(pNewBitmap.get(), rgbaData.data(), uploadWidth, uploadHeight, MapGLFilterToSDL(uiFilter),
+                             MapGLWrapToSDL(uiWrapMode), upscaled))
     {
         g_ErrorReport.Write(L"SDL texture upload failed %ls (%d)\r\n", filename.c_str(), uiBitmapIndex);
         return false;
     }
-    if (scale > Render::Textures::Upscale::NoScale)
+    pNewBitmap->UploadedBytes = UploadedBytes(uploadWidth, uploadHeight);
+    m_dwUsedTextureMemory += pNewBitmap->UploadedBytes;
+    if (!keepPixels)
     {
-        m_dwUsedTextureMemory -= static_cast<std::uint32_t>(ReleaseUploadedPixels(*pNewBitmap));
+        ReleaseUploadedPixels(*pNewBitmap);
     }
 
     SDL_GPUTexture* rawTexture = pNewBitmap->sdlTexture;
@@ -986,7 +1022,8 @@ bool CGlobalBitmap::OpenJpegTurbo(GLuint uiBitmapIndex, const std::wstring& file
     return true;
 }
 
-bool CGlobalBitmap::OpenTga(GLuint uiBitmapIndex, const std::wstring& filename, GLuint uiFilter, GLuint uiWrapMode, bool padToPowerOfTwo, bool premultiplyAlpha)
+bool CGlobalBitmap::OpenTga(GLuint uiBitmapIndex, const std::wstring& filename, GLuint uiFilter, GLuint uiWrapMode,
+                            bool padToPowerOfTwo, bool premultiplyAlpha, bool keepPixels)
 {
     namespace Ozt = Render::Textures::Ozt;
 
@@ -1003,46 +1040,60 @@ bool CGlobalBitmap::OpenTga(GLuint uiBitmapIndex, const std::wstring& filename,
     std::vector<unsigned char> pakBuffer((std::istreambuf_iterator<char>(input)), std::istreambuf_iterator<char>());
     input.close();
 
+    // A marked file holds an image up to its scale times the largest original.
+    const int markedScale = Render::Textures::Upscale::ReadMarkedScale(pakBuffer);
+    const int maxWidth = MAX_WIDTH * markedScale;
+    const int maxHeight = MAX_HEIGHT * markedScale;
+
     // A file cut off in its header or its pixels is too small.
-    const Ozt::Header header = Ozt::ReadHeader(pakBuffer, MAX_WIDTH, MAX_HEIGHT);
+    const Ozt::Header header = Ozt::ReadHeader(pakBuffer, maxWidth, maxHeight);
     if (header.status == Ozt::Status::TooSmall)
     {
         g_ErrorReport.Write(L"OpenTga: file too small %ls (%zu bytes)\r\n", filename_ozt.c_str(), pakBuffer.size());
         return false;
     }
     if (header.status == Ozt::Status::InvalidFormat)
     {
-        g_ErrorReport.Write(L"OpenTga: invalid format %ls (bit=%d, %dx%d)\r\n", filename_ozt.c_str(),
-                            header.bitsPerPixel, header.width, header.height);
+        g_ErrorReport.Write(L"OpenTga: invalid format %ls (bit=%d, %dx%d, at most %dx%d)\r\n", filename_ozt.c_str(),
+                            header.bitsPerPixel, header.width, header.height, maxWidth, maxHeight);
         return false;
     }
 
-    const int Width = padToPowerOfTwo ? NextPowerOfTwo(header.width, MAX_WIDTH) : header.width;
-    const int Height = padToPowerOfTwo ? NextPowerOfTwo(header.height, MAX_HEIGHT) : header.height;
+    const int Width = padToPowerOfTwo ? NextPowerOfTwo(header.width, maxWidth) : header.width;
+    const int Height = padToPowerOfTwo ? NextPowerOfTwo(header.height, maxHeight) : header.height;
     const int scale = Render::Textures::Upscale::ReadScale(pakBuffer, Width, Height);
+    const bool upscaled = scale > Render::Textures::Upscale::NoScale && !keepPixels;
 
     auto pNewBitmap = CreateBitmapStorage(uiBitmapIndex, filename, Width, Height, 4);
     pNewBitmap->Width = static_cast<float>(Width / scale);
     pNewBitmap->Height = static_cast<float>(Height / scale);
     const std::size_t BufferSize = pNewBitmap->BufferStorage.size();
-    m_dwUsedTextureMemory += static_cast<std::uint32_t>(BufferSize);
 
     Ozt::CopyPixels(pakBuffer, header, std::span<std::uint8_t>(pNewBitmap->Buffer, BufferSize), Width);
     if (premultiplyAlpha)
     {
         Render::RmlUi::PremultiplyAlpha(pNewBitmap->Buffer, BufferSize / Ozt::BytesPerPixel);
     }
 
+    int uploadWidth = Width;
+    int uploadHeight = Height;
+    if (keepPixels && scale > Render::Textures::Upscale::NoScale)
+    {
+        ShrinkCanvas(*pNewBitmap, uploadWidth, uploadHeight, scale);
+    }
+
     pNewBitmap->TextureNumber = uiBitmapIndex;
-    if (!UploadTextureSDLGpu(pNewBitmap.get(), pNewBitmap->Buffer, Width, Height, MapGLFilterToSDL(uiFilter),
-                             MapGLWrapToSDL(uiWrapMode), scale > Render::Textures::Upscale::NoScale))
+    if (!UploadTextureSDLGpu(pNewBitmap.get(), pNewBitmap->Buffer, uploadWidth, uploadHeight,
+                             MapGLFilterToSDL(uiFilter), MapGLWrapToSDL(uiWrapMode), upscaled))
     {
         g_ErrorReport.Write(L"SDL texture upload failed %ls (%d)\r\n", filename.c_str(), uiBitmapIndex);
         return false;
     }
-    if (scale > Render::Textures::Upscale::NoScale)
+    pNewBitmap->UploadedBytes = UploadedBytes(uploadWidth, uploadHeight);
+    m_dwUsedTextureMemory += pNewBitmap->UploadedBytes;
+    if (!keepPixels)
     {
-        m_dwUsedTextureMemory -= static_cast<std::uint32_t>(ReleaseUploadedPixels(*pNewBitmap));
+        ReleaseUploadedPixels(*pNewBitmap);
     }
 
     SDL_GPUTexture* rawTexture = pNewBitmap->sdlTexture;

src/source/Render/Sprites/GlobalBitmap.h

@@ -24,15 +24,16 @@ struct BITMAP_t
     GLuint BitmapIndex;
     wchar_t FileName[MAX_BITMAP_FILE_NAME];
     // The size the game works with. For upscaled art (Render/Textures/UpscaleMarker.h)
-    // the uploaded texture is larger and Buffer is released after the upload. A bitmap
-    // drawn on through Buffer (BITMAP_FONT, BITMAP_GUILD) must therefore never be upscaled.
+    // the uploaded texture is larger.
     float Width;
     float Height;
     char Components;
     GLuint TextureNumber;
     std::uint8_t Ref;
     bool IsSkin;
     bool IsHair;
+    // The pixels, released after the upload except for a canvas (LoadCanvasBitmap:
+    // BITMAP_GUILD, BITMAP_INTERFACE_MAP), which keeps them at the size above.
     BYTE* Buffer;
     std::vector<std::uint8_t> BufferStorage;
 
@@ -44,6 +45,8 @@ struct BITMAP_t
 #pragma pack(pop)
     SDL_GPUTexture* sdlTexture = nullptr;
     SDL_GPUSampler* sdlSampler = nullptr;
+    // The base level, counted as texture memory.
+    std::uint32_t UploadedBytes = 0;
 };
 
 class CBitmapCache
@@ -131,7 +134,11 @@ class CGlobalBitmap
     virtual ~CGlobalBitmap();
 
     GLuint LoadImage(const std::wstring& filename, GLuint uiFilter = GL_NEAREST, GLuint uiWrapMode = GL_CLAMP_TO_EDGE);
-    bool LoadImage(GLuint uiBitmapIndex, const std::wstring& filename, GLuint uiFilter = GL_NEAREST, GLuint uiWrapMode = GL_CLAMP_TO_EDGE, bool padToPowerOfTwo = true, bool premultiplyAlpha = false);
+    // keepPixels: the game draws on the texture through Buffer (a canvas), so Buffer stays
+    // at the original size, also for an upscaled file.
+    bool LoadImage(GLuint uiBitmapIndex, const std::wstring& filename, GLuint uiFilter = GL_NEAREST,
+                   GLuint uiWrapMode = GL_CLAMP_TO_EDGE, bool padToPowerOfTwo = true, bool premultiplyAlpha = false,
+                   bool keepPixels = false);
     // Like LoadImage(filename, ...), but never shares an existing by-filename match -- always
     // creates its own independent entry, even if the same file is already resident under a
     // numbered slot (e.g. BITMAP_LOG_IN+7). Needed by callers that cache the resolved GPU
@@ -180,8 +187,11 @@ class CGlobalBitmap
     GLuint GenerateTextureIndex();
     GLuint FindAvailableTextureIndex(GLuint uiSeed);
 
-    bool OpenJpegTurbo(GLuint uiBitmapIndex, const std::wstring& filename, GLuint uiFilter = GL_NEAREST, GLuint uiWrapMode = GL_CLAMP_TO_EDGE, bool padToPowerOfTwo = true);
-    bool OpenTga(GLuint uiBitmapIndex, const std::wstring& filename, GLuint uiFilter = GL_NEAREST, GLuint uiWrapMode = GL_CLAMP_TO_EDGE, bool padToPowerOfTwo = true, bool premultiplyAlpha = false);
+    bool OpenJpegTurbo(GLuint uiBitmapIndex, const std::wstring& filename, GLuint uiFilter = GL_NEAREST,
+                       GLuint uiWrapMode = GL_CLAMP_TO_EDGE, bool padToPowerOfTwo = true, bool keepPixels = false);
+    bool OpenTga(GLuint uiBitmapIndex, const std::wstring& filename, GLuint uiFilter = GL_NEAREST,
+                 GLuint uiWrapMode = GL_CLAMP_TO_EDGE, bool padToPowerOfTwo = true, bool premultiplyAlpha = false,
+                 bool keepPixels = false);
     void SplitFileName(IN const std::wstring& filepath, OUT std::wstring& filename, bool bIncludeExt);
     void SplitExt(IN const std::wstring& filepath, OUT std::wstring& ext, bool bIncludeDot);
     void ExchangeExt(IN const std::wstring& in_filepath, IN const std::wstring& ext, OUT std::wstring& out_filepath);

src/source/Render/Textures/UpscaleMarker.cpp

@@ -19,18 +19,55 @@ bool HasMarker(std::span<const std::uint8_t> file)
 }
 } // namespace
 
-int ReadScale(std::span<const std::uint8_t> file, int textureWidth, int textureHeight)
+int ReadMarkedScale(std::span<const std::uint8_t> file)
 {
     if (!HasMarker(file))
     {
         return NoScale;
     }
 
     const int scale = file[ScaleOffset];
-    if (!IsSupportedScale(scale) || textureWidth < scale || textureHeight < scale)
+    return IsSupportedScale(scale) ? scale : NoScale;
+}
+
+int ReadScale(std::span<const std::uint8_t> file, int textureWidth, int textureHeight)
+{
+    const int scale = ReadMarkedScale(file);
+    if (textureWidth < scale || textureHeight < scale)
     {
         return NoScale;
     }
     return scale;
 }
+
+std::vector<std::uint8_t> ShrinkToOriginal(std::span<const std::uint8_t> pixels, int width, int height,
+                                           int bytesPerPixel, int scale)
+{
+    const int originalWidth = width / scale;
+    const int originalHeight = height / scale;
+    const int blockPixels = scale * scale;
+    std::vector<std::uint8_t> original(static_cast<std::size_t>(originalWidth) * originalHeight * bytesPerPixel);
+
+    for (int y = 0; y < originalHeight; ++y)
+    {
+        for (int x = 0; x < originalWidth; ++x)
+        {
+            for (int channel = 0; channel < bytesPerPixel; ++channel)
+            {
+                int sum = 0;
+                for (int blockY = 0; blockY < scale; ++blockY)
+                {
+                    const std::size_t row = static_cast<std::size_t>(y * scale + blockY) * width;
+                    for (int blockX = 0; blockX < scale; ++blockX)
+                    {
+                        sum += pixels[(row + x * scale + blockX) * bytesPerPixel + channel];
+                    }
+                }
+                original[(static_cast<std::size_t>(y) * originalWidth + x) * bytesPerPixel + channel] =
+                    static_cast<std::uint8_t>(sum / blockPixels);
+            }
+        }
+    }
+    return original;
+}
 } // namespace Render::Textures::Upscale