Prototype of RAW decoding in Skia.

src/codec/SkRawCodec.cpp

+/*
+ * Copyright 2015 Google Inc.
+ *
+ * Use of this source code is governed by a BSD-style license that can be
+ * found in the LICENSE file.
+ */
+
+#include "SkCodec.h"
+#include "SkCodecPriv.h"
+#include "SkColorPriv.h"
+#include "SkData.h"
+#if !defined(GOOGLE3)
+#include "SkJpegCodec.h"
+#endif
+#include "SkRawCodec.h"
+#include "SkRefCnt.h"
+#include "SkStream.h"
+#include "SkStreamPriv.h"
+#include "SkSwizzler.h"
+#include "SkTemplates.h"
+#include "SkTypes.h"
+
+#include "dng_color_space.h"
+#include "dng_exceptions.h"
+#include "dng_host.h"
+#include "dng_info.h"
+#include "dng_render.h"
+#include "dng_stream.h"
+
+#include "src/piex.h"
+
+#include <cmath>  // for std::round,floor,ceil
+#include <limits>
+
+namespace {
+
+// T must be unsigned type.
+template <class T>
+bool safe_add_to_size_t(T arg1, T arg2, size_t* result) {
+    SkASSERT(arg1 >= 0);
+    SkASSERT(arg2 >= 0);
+    if (arg1 >= 0 && arg2 <= std::numeric_limits<T>::max() - arg1) {
+        T sum = arg1 + arg2;
+        if (sum <= std::numeric_limits<size_t>::max()) {
+            *result = static_cast<size_t>(sum);
+            return true;
+        }
+    }
+    return false;
+}
+
+}  // namespace
+
+// Note: this class could throw exception if it is used as dng_stream.
+class SkRawStream : public dng_stream, public ::piex::StreamInterface {
+public:
+    // Note that this call will take the ownership of stream.
+    explicit SkRawStream(SkStream* stream)
+        : fStream(stream), fWholeStreamRead(false) {}
+
+    ~SkRawStream() override {}
+
+    /*
+     * Creates a SkMemoryStream from the offset with size.

[scroggo, 2016/01/14 15:26:38]

nit: an SkMemoryStream

[yujieqin, 2016/01/15 14:49:31]

Done.

+     * Note: for performance reason, this function is destructive to the SkRawStream. One should
+     *       abandon current object after the function call.
+     */
+    SkMemoryStream* copyBuffer(size_t offset, size_t size) {

[scroggo, 2016/01/14 15:26:38]

Maybe this should have a more descriptive name that copyBuffer. It no longer
necessarily copies a buffer, but it does create an SkMemoryStream and destroy
the SkRawStream. That said, I do not have an idea for a better name.

[yujieqin, 2016/01/15 14:49:30]

Let's call it transferBuffer().

+        SkAutoTUnref<SkData> data(SkData::NewUninitialized(size));
+        if (offset > fStreamBuffer.bytesWritten()) {
+            // If the offset is not buffered, read from fStream directly and skip the buffering.
+            // This is destructive to currect object.
+            const size_t skipLength = offset - fStreamBuffer.bytesWritten();
+            if (fStream->skip(skipLength) != skipLength) {
+                return nullptr;
+            }
+            if (fStream->read(data->writable_data(), size) != size) {
+                return nullptr;

[scroggo, 2016/01/14 15:26:38]

I suggested this, and it matches the intent of the code you originally wrote.
But it occurred to me that we generally will decode incomplete images. If we
want to display the partial JPEG (we won't know how complete it is until
SkJpegCodec gets a hold of it) we can do the following:

const size_t bytesRead = fStream->read(data->writable_data(), size));
if (bytesRead < size) {
    data.reset(SkData::NewSubset(data.get(), 0, bytesRead));
}

[yujieqin, 2016/01/15 14:49:30]

Done.

+            }
+        } else {
+            const size_t alreadyBuffered = fStreamBuffer.bytesWritten() - offset;
+            if (!fStreamBuffer.read(data->writable_data(), offset, alreadyBuffered)) {
+                return nullptr;
+            }
+
+            const size_t remaining = size - alreadyBuffered;
+            auto* dst = static_cast<uint8_t*>(data->writable_data()) + alreadyBuffered;
+            if (remaining && fStream->read(dst, remaining) != remaining) {

[scroggo, 2016/01/14 15:26:38]

The above comment ("This is destructive to the current object") is true here as
well.

[yujieqin, 2016/01/15 14:49:31]

true, I just removed the line 72, and we had comment about this of the method
already.

+                return nullptr;

[scroggo, 2016/01/14 15:26:38]

Similar to the above case, we can pass what is available to SkJpegCodec if we
read less than "remaining" bytes.

[yujieqin, 2016/01/15 14:49:30]

Done.

+            }
+        }
+        return new SkMemoryStream(data);
+    }
+
+    // For PIEX
+    ::piex::Error GetData(const size_t offset, const size_t length,
+                          uint8* data) override {
+        if (offset == 0 && length == 0) {
+            return ::piex::Error::kOk;
+        }
+        size_t sum;
+        if (!safe_add_to_size_t(offset, length, &sum) || !this->bufferMoreData(sum)) {
+            return ::piex::Error::kFail;
+        }
+        if (!fStreamBuffer.read(data, offset, length)) {
+            return ::piex::Error::kFail;
+        }
+        return ::piex::Error::kOk;
+    }
+
+protected:
+    // For dng_stream
+    uint64 DoGetLength() override {
+        if (!this->bufferMoreData(kReadToEnd)) {  // read whole stream
+            ThrowReadFile();
+        }
+        return fStreamBuffer.bytesWritten();
+    }
+
+    // For dng_stream
+    void DoRead(void* data, uint32 count, uint64 offset) override {
+        if (count == 0 && offset == 0) {
+            return;
+        }
+        size_t sum;
+        if (!safe_add_to_size_t(static_cast<uint64>(count), offset, &sum) ||
+            !this->bufferMoreData(sum)) {
+            ThrowReadFile();
+        }
+
+        if (!fStreamBuffer.read(data, offset, count)) {
+            ThrowReadFile();
+        }
+    }
+
+private:
+    // Note: if the newSize == kReadToEnd (0), this function will read to the end of stream.
+    bool bufferMoreData(size_t newSize) {
+        if (newSize == kReadToEnd) {
+            if (fWholeStreamRead) {  // already read-to-end.
+                return true;
+            }
+
+            // TODO: optimize for the special case when the input is SkMemoryStream.
+            return SkStreamCopy(&fStreamBuffer, fStream.get());
+        }
+
+        if (newSize <= fStreamBuffer.bytesWritten()) {  // already buffered to newSize
+            return true;
+        }
+        if (fWholeStreamRead) {  // newSize is larger than the whole stream.
+            return false;
+        }
+
+        const size_t sizeToRead = newSize - fStreamBuffer.bytesWritten();
+        SkAutoTMalloc<uint8> tempBuffer(sizeToRead);
+        const size_t bytesRead = fStream->read(tempBuffer.get(), sizeToRead);
+        if (bytesRead != sizeToRead) {
+            return false;
+        }
+        return fStreamBuffer.write(tempBuffer.get(), bytesRead);
+    }
+
+    SkAutoTDelete<SkStream> fStream;
+    bool fWholeStreamRead;
+
+    SkDynamicMemoryWStream fStreamBuffer;
+
+    const size_t kReadToEnd = 0;
+};
+
+class SkDngImage {
+public:
+    static SkDngImage* NewFromStream(SkRawStream* stream) {
+        SkAutoTDelete<dng_host> host(new dng_host);
+        SkAutoTDelete<dng_info> info(new dng_info);
+        SkAutoTDelete<dng_negative> negative(SkDngImage::ReadDng(stream, host.get(), info.get()));
+        if (!negative) {
+            return nullptr;
+        }
+        return new SkDngImage(stream, host.release(), info.release(), negative.release());
+    }
+
+    /*
+     * Renders the DNG image to the size.
+     * The rendered image will be close to the specified size, but there is no guarantee that any

[scroggo, 2016/01/14 15:26:38]

So long as the specified size is greater than 128 on the long edge (DNG SDK bug
###)

[yujieqin, 2016/01/15 14:49:31]

We update the comment, but we did not add a bug for DNG SDK, because we don't
think there is anything to do on the DNG SDK side.

+     * of the edges will match the requested size. E.g.
+     *   100% size:              4000 x 3000
+     *   requested size:         1600 x 1200
+     *   returned size could be: 2000 x 1500
+     */
+    dng_image* render(int width, int height) {
+        // render() takes the ownership of fHost, fInfo and fNegative when available.

[scroggo, 2016/01/14 15:26:38]

nit: takes ownership* (no need for "the" here)

[yujieqin, 2016/01/15 14:49:31]

Done.

[yujieqin, 2016/01/15 14:49:31]

Done.

+        SkAutoTDelete<dng_host> host;
+        SkAutoTDelete<dng_info> info;
+        SkAutoTDelete<dng_negative> negative;
+        if (!fHost || !fInfo || !fNegative) {
+            SkCodecPrintf("Warning: SkDngImage::render() is called multiple times.");
+            host.reset(new dng_host);
+            info.reset(new dng_info);
+            negative.reset(SkDngImage::ReadDng(fStream.get(), host.get(), info.get()));
+            if (!negative) {
+                return nullptr;
+            }
+        } else {
+            host.reset(fHost.release());
+            info.reset(fInfo.release());
+            negative.reset(fNegative.release());
+        }
+
+        // DNG SDK preserves the aspect ratio, so it only needs to know the longer dimension.
+        const int preferredSize = SkTMax(width, height);
+        try {
+            host->SetPreferredSize(preferredSize);
+            host->ValidateSizes();
+
+            negative->ReadStage1Image(*host, *fStream, *info);
+
+            if (info->fMaskIndex != -1) {
+                negative->ReadTransparencyMask(*host, *fStream, *info);
+            }
+
+            negative->ValidateRawImageDigest(*host);
+            if (negative->IsDamaged()) {
+                return nullptr;
+            }
+
+            const int32 kMosaicPlane = -1;
+            negative->BuildStage2Image(*host);
+            negative->BuildStage3Image(*host, kMosaicPlane);
+
+            dng_render render(*host, *negative);
+            render.SetFinalSpace(dng_space_sRGB::Get());
+            render.SetFinalPixelType(ttByte);
+
+            dng_point stage3_size = negative->Stage3Image()->Size();
+            render.SetMaximumSize(SkTMax(stage3_size.h, stage3_size.v));
+
+            return render.Render();
+        } catch (...) {
+            return nullptr;
+        }
+    }
+
+    SkImageInfo getImageInfo() const {

[scroggo, 2016/01/14 15:26:38]

nit: You can make this return const SkImageInfo& to avoid a copy.

[yujieqin, 2016/01/15 14:49:31]

Done.

+        return fImageInfo;
+    }
+
+private:
+    static dng_negative* ReadDng(SkRawStream* stream, dng_host* host, dng_info* info) {
+        try {
+            host->ValidateSizes();
+            info->Parse(*host, *stream);
+            info->PostParse(*host);
+            if (!info->IsValidDNG()) {
+                return nullptr;
+            }
+
+            SkAutoTDelete<dng_negative> negative;
+            negative.reset(host->Make_dng_negative());
+            negative->Parse(*host, *stream, *info);
+            negative->PostParse(*host, *stream, *info);
+            negative->SynchronizeMetadata();
+            return negative.release();
+        } catch (...) {
+            return nullptr;
+        }
+    }
+
+    SkDngImage(SkRawStream* stream, dng_host* host, dng_info* info, dng_negative* negative)
+        : fStream(stream)
+        , fHost(host)
+        , fInfo(info)
+        , fNegative(negative)
+        , fImageInfo(SkImageInfo::Make(fNegative->DefaultCropSizeH().As_real64(),
+                                       fNegative->DefaultCropSizeV().As_real64(),
+                                       kN32_SkColorType, kOpaque_SkAlphaType)) {}
+
+    SkAutoTDelete<SkRawStream> fStream;
+    SkAutoTDelete<dng_host> fHost;
+    SkAutoTDelete<dng_info> fInfo;
+    SkAutoTDelete<dng_negative> fNegative;
+
+    const SkImageInfo fImageInfo;
+};
+
+/*
+ * Tries to handle the image with PIEX. If PIEX returns kOk and finds the preview image, create a
+ * SkJpegCodec. If PIEX returns kFail, then the file is invalid, return nullptr. In other cases,
+ * fallback to create SkRawCodec for DNG images.
+ */
+SkCodec* SkRawCodec::NewFromStream(SkStream* stream) {
+    SkAutoTDelete<SkRawStream> rawStream(new SkRawStream(stream));
+    ::piex::PreviewImageData imageData;
+    if (::piex::IsRaw(rawStream.get())) {
+        ::piex::Error error = ::piex::GetPreviewImageData(rawStream.get(), &imageData);
+
+        if (error == ::piex::Error::kOk && imageData.preview_length > 0) {
+#if !defined(GOOGLE3)
+            // copyBuffer() is destructive to the rawStream. Abandon the rawStream after this
+            // function call.
+            SkMemoryStream* memoryStream =
+                    rawStream->copyBuffer(imageData.preview_offset, imageData.preview_length);
+            return memoryStream ? SkJpegCodec::NewFromStream(memoryStream) : nullptr;
+#else
+            return nullptr;
+#endif
+        } else if (error == ::piex::Error::kFail) {
+            return nullptr;
+        }
+    }
+
+    SkAutoTDelete<SkDngImage> dngImage(SkDngImage::NewFromStream(rawStream.release()));
+    if (!dngImage) {
+        return nullptr;
+    }
+
+    return new SkRawCodec(dngImage.release());
+}
+
+SkCodec::Result SkRawCodec::onGetPixels(const SkImageInfo& requestedInfo, void* dst,

[scroggo, 2016/01/14 15:26:38]

This function needs to check to see that the requestedInfo's colortype is okay.
For other codecs, we do this in conversion_possible in SkCodecPriv.h

[yujieqin, 2016/01/15 14:49:30]

The DNG SDK returns RGB 8bit data. Currently we are using the SkSwizzler to
convert it to requested info and options. Could you be more specified with the
check you are mentioning? 

[scroggo, 2016/01/19 21:05:44]

The check is to early exit. Yes, the SkSwizzler does the conversion, but you
assert that you have created an SkSwizzler. If the caller requested kIndex8, the
SkSwizzler will be NULL. We could rely on that happening (but we'd need to
change the assert to a branch). Elsewhere we call conversion_supported; if it
returns false, we never attempt to create the swizzler. Look at that method in
SkCodecPriv.h (it's short and straightforward) and where it's called (in the
beginning of onGetPixels in other codec implementations). The beginning of this
method should have the same call.

[yujieqin, 2016/01/20 12:53:03]

I see your point. Added the conversion_possible() check and the check to
(!swizzler).

[scroggo, 2016/01/20 18:13:35]

With conversion_possible exiting early, you may not need the check for the
swizzler. (I don't remember for sure; there was a point when we needed to check,
before we had implemented many of them/before conversion_supported existed, but
we still do check in some other codecs. Matt, do you remember?)

[yujieqin, 2016/01/20 21:42:19]

From my test with DM, this is still necessary, because I run into some cases
here "swizzler == nullptr" using our test images. 

+                                        size_t dstRowBytes, const Options& options,
+                                        SkPMColor ctable[], int* ctableCount,
+                                        int* rowsDecoded) {
+    int width = requestedInfo.width();
+    int height = requestedInfo.height();
+    SkAutoTDelete<dng_image> image(fDngImage->render(width, height));
+    if (!image) {
+        return kInvalidInput;
+    }
+
+    // Because the DNG SDK can not gaurantee to render to requested size, we allow a small

[scroggo, 2016/01/14 15:26:38]

guarantee*

[yujieqin, 2016/01/15 14:49:30]

Done.

+    // difference. Only the overlapping region will be copied in onGetPixels().

[scroggo, 2016/01/14 15:26:38]

nit: It's not really a copy - it looks like it's a two step conversion:
- from the format of dng_image to srcRow
- from srcRow to the output memory

Which reminds me, would it be possible to skip one of these steps? i.e. can we
convert directly from the dng_image to the output memory, either using options
of dng_image or by adding modes to the swizzler?

[yujieqin, 2016/01/15 14:49:30]

It seems like we can not avoid these two step conversions. Because the DNG SDK
is lack of options for the conversions. Depending on your other question about
checking the requestedInfo's colortype, we might be able to do something
different. Could you postpone this optimization after we got the first round of
this CL submitted?

[scroggo, 2016/01/19 21:05:44]

It's fine if we cannot avoid the two step conversion. I just speculated that it
might be possible, not knowing anything about DNG SDK. I had two ideas, which
aren't directly related to my comment about requestedInfo's colorType:

1. Convert directly to 32 bit (or 565) in DNG SDK
Again, I do not know enough about DNG SDK to know whether this is possible.
(libjpeg-turbo lets us convert directly to 565, and I think 32 bit, meaning we
do not need an SkSwizzler if we are not sampling.) Even if it is possible, that
does not mean we should do it. (For example, libpng can do this, but it's slower
than what we would do otherwise.)

2. Convert directly from the image's native format
Right now, you convert from whatever the image is (I think you said it might be
16 bits per component, for example?) into RGB (with 8 bits per component), which
the SkSwizzler converts into what we want. But we could also theoretically add a
new type to SkSwizzler::SrcConfig which specifies another input format (like 16
bits per component). Then we could convert directly from the source type in the
SkSwizzler (for which Matt is writing SIMD optimizations). Does that make sense?

This does not need to prevent this CL from landing, but I could imagine that it
might be beneficial.

[yujieqin, 2016/01/20 12:53:04]

AFAIK,

For option 1, it is not possible due to the limit of DNG SDK.
For option 2, due to the general structure of DNG SDK, it is also not possible.
But we could investigate more later.

+    const float maxDiffRatio = 1.02f;
+    const dng_point& imageSize = image->Size();
+    if (SkTMax(width, imageSize.h) / SkTMin(width, imageSize.h) > maxDiffRatio ||

[msarett, 2016/01/14 19:02:14]

If width is greater than imageSize.h and/or height is greater than imageSize.v,
will we leave dst memory uninitialized?

Even if we fill it with something, won't that look weird?

[yujieqin, 2016/01/15 14:49:30]

Changed logic, we don't have this case anymore.

+        SkTMax(height, imageSize.v) / SkTMin(height, imageSize.v) > maxDiffRatio) {
+        return SkCodec::kInvalidScale;
+    }
+
+    // Only copy the top-left of the overlapping region.

[scroggo, 2016/01/14 15:26:38]

Again, I think "copy" is misleading here.

[yujieqin, 2016/01/15 14:49:30]

we don't need this anymore.

+    width = SkTMin(width, imageSize.h);
+    height = SkTMin(height, imageSize.v);
+
+    void* dstRow = dst;
+    uint8_t srcRow[width * 3];
+    SkAutoTDelete<SkSwizzler> swizzler(SkSwizzler::CreateSwizzler(
+            SkSwizzler::kRGB, nullptr, requestedInfo, options));
+    SkASSERT(swizzler);
+
+    dng_pixel_buffer buffer;
+    buffer.fData = &srcRow[0];
+    buffer.fPlane = 0;
+    buffer.fPlanes = 3;
+    buffer.fColStep = buffer.fPlanes;
+    buffer.fPlaneStep = 1;
+    buffer.fPixelType = ttByte;
+    buffer.fPixelSize = sizeof(uint8_t);
+    buffer.fRowStep = sizeof(srcRow);
+
+    for (int i = 0; i < height; ++i) {
+        buffer.fArea = dng_rect(i, 0, i + 1, width);
+
+        try {
+            image->Get(buffer, dng_image::edge_zero);
+        } catch (...) {
+            *rowsDecoded = i;
+            return kIncompleteInput; 
+        }
+
+        swizzler->swizzle(dstRow, &srcRow[0]);
+        dstRow = SkTAddOffset<void>(dstRow, dstRowBytes);
+    }
+    return kSuccess;
+}
+
+SkISize SkRawCodec::onGetScaledDimensions(float desiredScale) const {
+    SkASSERT(desiredScale <= 1.f);
+    SkISize dim = this->getInfo().dimensions();
+    const int longEdge = SkTMax(dim.fWidth, dim.fHeight);
+
+    // Limits the minimun size to be 128 on the long edge.
+    if (desiredScale < 128.f / static_cast<float>(longEdge)) {
+        desiredScale = 128.f / static_cast<float>(longEdge);
+    }
+
+    const float finalScale = (int)(1.f/ desiredScale);
+
+    dim.fWidth = std::round(dim.fWidth / finalScale);
+    dim.fHeight = std::round(dim.fHeight / finalScale);
+    return dim;
+}
+
+bool SkRawCodec::onDimensionsSupported(const SkISize& dim) {
+    const SkISize fullDim = this->getInfo().dimensions();
+    const float fullLongEdge = SkTMax(fullDim.fWidth, fullDim.fHeight);
+    const float longEdge = SkTMax(dim.fWidth, dim.fHeight);
+
+    SkISize sizeFloor = this->onGetScaledDimensions(1.f / std::floor(fullLongEdge / longEdge));
+    SkISize sizeCeil = this->onGetScaledDimensions(1.f / std::ceil(fullLongEdge / longEdge));
+    return sizeFloor == dim || sizeCeil == dim;
+}
+
+SkRawCodec::~SkRawCodec() {}
+
+SkRawCodec::SkRawCodec(SkDngImage* dngImage)
+    : INHERITED(dngImage->getImageInfo(), nullptr)
+    , fDngImage(dngImage) {}