xref: /third_party/skia/src/codec/SkHeifCodec.cpp (revision cb93a386) 
1/*
2 * Copyright 2017 Google Inc.
3 *
4 * Use of this source code is governed by a BSD-style license that can be
5 * found in the LICENSE file.
6 */
7
8#include "include/core/SkTypes.h"
9
10#ifdef SK_HAS_HEIF_LIBRARY
11#include "include/codec/SkCodec.h"
12#include "include/core/SkStream.h"
13#include "include/private/SkColorData.h"
14#include "src/codec/SkCodecPriv.h"
15#include "src/codec/SkHeifCodec.h"
16#include "src/core/SkEndian.h"
17
18#if !defined(WIN32) && !defined(_WIN32) && !defined(__WIN32__)
19#include <dlfcn.h>
20#endif
21
22#define FOURCC(c1, c2, c3, c4) \
23    ((c1) << 24 | (c2) << 16 | (c3) << 8 | (c4))
24
25const static char *HEIF_IMPL_SO_NAME = "libheifimpl.z.so";
26
27bool SkHeifCodec::IsSupported(const void* buffer, size_t bytesRead,
28                              SkEncodedImageFormat* format) {
29    // Parse the ftyp box up to bytesRead to determine if this is HEIF or AVIF.
30    // Any valid ftyp box should have at least 8 bytes.
31    if (bytesRead < 8) {
32        return false;
33    }
34
35    uint32_t* ptr = (uint32_t*)buffer;
36    uint64_t chunkSize = SkEndian_SwapBE32(ptr[0]);
37    uint32_t chunkType = SkEndian_SwapBE32(ptr[1]);
38
39    if (chunkType != FOURCC('f', 't', 'y', 'p')) {
40        return false;
41    }
42
43    int64_t offset = 8;
44    if (chunkSize == 1) {
45        // This indicates that the next 8 bytes represent the chunk size,
46        // and chunk data comes after that.
47        if (bytesRead < 16) {
48            return false;
49        }
50        auto* chunkSizePtr = SkTAddOffset<const uint64_t>(buffer, offset);
51        chunkSize = SkEndian_SwapBE64(*chunkSizePtr);
52        if (chunkSize < 16) {
53            // The smallest valid chunk is 16 bytes long in this case.
54            return false;
55        }
56        offset += 8;
57    } else if (chunkSize < 8) {
58        // The smallest valid chunk is 8 bytes long.
59        return false;
60    }
61
62    if (chunkSize > bytesRead) {
63        chunkSize = bytesRead;
64    }
65    int64_t chunkDataSize = chunkSize - offset;
66    // It should at least have major brand (4-byte) and minor version (4-bytes).
67    // The rest of the chunk (if any) is a list of (4-byte) compatible brands.
68    if (chunkDataSize < 8) {
69        return false;
70    }
71
72    uint32_t numCompatibleBrands = (chunkDataSize - 8) / 4;
73    bool isHeif = false;
74    for (size_t i = 0; i < numCompatibleBrands + 2; ++i) {
75        if (i == 1) {
76            // Skip this index, it refers to the minorVersion,
77            // not a brand.
78            continue;
79        }
80        auto* brandPtr = SkTAddOffset<const uint32_t>(buffer, offset + 4 * i);
81        uint32_t brand = SkEndian_SwapBE32(*brandPtr);
82        if (brand == FOURCC('m', 'i', 'f', '1') || brand == FOURCC('h', 'e', 'i', 'c')
83         || brand == FOURCC('m', 's', 'f', '1') || brand == FOURCC('h', 'e', 'v', 'c')
84         || brand == FOURCC('a', 'v', 'i', 'f') || brand == FOURCC('a', 'v', 'i', 's')) {
85            // AVIF files could have "mif1" as the major brand. So we cannot
86            // distinguish whether the image is AVIF or HEIC just based on the
87            // "mif1" brand. So wait until we see a specific avif brand to
88            // determine whether it is AVIF or HEIC.
89            isHeif = true;
90            if (brand == FOURCC('a', 'v', 'i', 'f')
91              || brand == FOURCC('a', 'v', 'i', 's')) {
92                if (format != nullptr) {
93                    *format = SkEncodedImageFormat::kAVIF;
94                }
95                return true;
96            }
97        }
98    }
99    if (isHeif) {
100        if (format != nullptr) {
101            *format = SkEncodedImageFormat::kHEIF;
102        }
103        return true;
104    }
105    return false;
106}
107
108static SkEncodedOrigin get_orientation(const HeifFrameInfo& frameInfo) {
109    switch (frameInfo.mRotationAngle) {
110        case 0:   return kTopLeft_SkEncodedOrigin;
111        case 90:  return kRightTop_SkEncodedOrigin;
112        case 180: return kBottomRight_SkEncodedOrigin;
113        case 270: return kLeftBottom_SkEncodedOrigin;
114    }
115    return kDefault_SkEncodedOrigin;
116}
117
118struct SkHeifStreamWrapper : public HeifStream {
119    SkHeifStreamWrapper(SkStream* stream) : fStream(stream) {}
120
121    ~SkHeifStreamWrapper() override {}
122
123    size_t read(void* buffer, size_t size) override {
124        return fStream->read(buffer, size);
125    }
126
127    bool rewind() override {
128        return fStream->rewind();
129    }
130
131    bool seek(size_t position) override {
132        return fStream->seek(position);
133    }
134
135    bool hasLength() const override {
136        return fStream->hasLength();
137    }
138
139    size_t getLength() const override {
140        return fStream->getLength();
141    }
142
143    bool hasPosition() const override {
144        return fStream->hasPosition();
145    }
146
147    size_t getPosition() const override {
148        return fStream->getPosition();
149    }
150
151private:
152    std::unique_ptr<SkStream> fStream;
153};
154
155static void releaseProc(const void* ptr, void* context) {
156    delete reinterpret_cast<std::vector<uint8_t>*>(context);
157}
158
159std::unique_ptr<SkCodec> SkHeifCodec::MakeFromStream(std::unique_ptr<SkStream> stream,
160        SkCodec::SelectionPolicy selectionPolicy, SkEncodedImageFormat format, Result* result) {
161    std::unique_ptr<HeifDecoder> heifDecoder(createHeifDecoder());
162    if (heifDecoder == nullptr) {
163        *result = kInternalError;
164        return nullptr;
165    }
166
167    HeifFrameInfo heifInfo;
168    if (!heifDecoder->init(new SkHeifStreamWrapper(stream.release()), &heifInfo)) {
169        *result = kInvalidInput;
170        return nullptr;
171    }
172
173    size_t frameCount = 1;
174    if (selectionPolicy == SkCodec::SelectionPolicy::kPreferAnimation) {
175        HeifFrameInfo sequenceInfo;
176        if (heifDecoder->getSequenceInfo(&sequenceInfo, &frameCount) &&
177                frameCount > 1) {
178            heifInfo = std::move(sequenceInfo);
179        }
180    }
181
182    std::unique_ptr<SkEncodedInfo::ICCProfile> profile = nullptr;
183    if (heifInfo.mIccData.size() > 0) {
184        auto iccData = new std::vector<uint8_t>(std::move(heifInfo.mIccData));
185        auto icc = SkData::MakeWithProc(iccData->data(), iccData->size(), releaseProc, iccData);
186        profile = SkEncodedInfo::ICCProfile::Make(std::move(icc));
187    }
188    if (profile && profile->profile()->data_color_space != skcms_Signature_RGB) {
189        // This will result in sRGB.
190        profile = nullptr;
191    }
192
193    SkEncodedInfo info = SkEncodedInfo::Make(heifInfo.mWidth, heifInfo.mHeight,
194            SkEncodedInfo::kYUV_Color, SkEncodedInfo::kOpaque_Alpha, 8, std::move(profile));
195    SkEncodedOrigin orientation = get_orientation(heifInfo);
196
197    *result = kSuccess;
198    return std::unique_ptr<SkCodec>(new SkHeifCodec(
199            std::move(info), heifDecoder.release(), orientation, frameCount > 1, format));
200}
201
202SkHeifCodec::SkHeifCodec(
203        SkEncodedInfo&& info,
204        HeifDecoder* heifDecoder,
205        SkEncodedOrigin origin,
206        bool useAnimation,
207        SkEncodedImageFormat format)
208    : INHERITED(std::move(info), skcms_PixelFormat_RGBA_8888, nullptr, origin)
209    , fHeifDecoder(heifDecoder)
210    , fSwizzleSrcRow(nullptr)
211    , fColorXformSrcRow(nullptr)
212    , fUseAnimation(useAnimation)
213    , fFormat(format)
214{}
215
216bool SkHeifCodec::conversionSupported(const SkImageInfo& dstInfo, bool srcIsOpaque,
217                                      bool needsColorXform) {
218    SkASSERT(srcIsOpaque);
219
220    if (kUnknown_SkAlphaType == dstInfo.alphaType()) {
221        return false;
222    }
223
224    if (kOpaque_SkAlphaType != dstInfo.alphaType()) {
225        SkCodecPrintf("Warning: an opaque image should be decoded as opaque "
226                "- it is being decoded as non-opaque, which will draw slower\n");
227    }
228
229    switch (dstInfo.colorType()) {
230        case kRGBA_8888_SkColorType:
231            return fHeifDecoder->setOutputColor(kHeifColorFormat_RGBA_8888);
232
233        case kBGRA_8888_SkColorType:
234            return fHeifDecoder->setOutputColor(kHeifColorFormat_BGRA_8888);
235
236        case kRGB_565_SkColorType:
237            if (needsColorXform) {
238                return fHeifDecoder->setOutputColor(kHeifColorFormat_RGBA_8888);
239            } else {
240                return fHeifDecoder->setOutputColor(kHeifColorFormat_RGB565);
241            }
242
243        case kRGBA_F16_SkColorType:
244            SkASSERT(needsColorXform);
245            return fHeifDecoder->setOutputColor(kHeifColorFormat_RGBA_8888);
246
247        default:
248            return false;
249    }
250}
251
252int SkHeifCodec::readRows(const SkImageInfo& dstInfo, void* dst, size_t rowBytes, int count,
253                          const Options& opts) {
254    // When fSwizzleSrcRow is non-null, it means that we need to swizzle.  In this case,
255    // we will always decode into fSwizzlerSrcRow before swizzling into the next buffer.
256    // We can never swizzle "in place" because the swizzler may perform sampling and/or
257    // subsetting.
258    // When fColorXformSrcRow is non-null, it means that we need to color xform and that
259    // we cannot color xform "in place" (many times we can, but not when the dst is F16).
260    // In this case, we will color xform from fColorXformSrcRow into the dst.
261    uint8_t* decodeDst = (uint8_t*) dst;
262    uint32_t* swizzleDst = (uint32_t*) dst;
263    size_t decodeDstRowBytes = rowBytes;
264    size_t swizzleDstRowBytes = rowBytes;
265    int dstWidth = opts.fSubset ? opts.fSubset->width() : dstInfo.width();
266    if (fSwizzleSrcRow && fColorXformSrcRow) {
267        decodeDst = fSwizzleSrcRow;
268        swizzleDst = fColorXformSrcRow;
269        decodeDstRowBytes = 0;
270        swizzleDstRowBytes = 0;
271        dstWidth = fSwizzler->swizzleWidth();
272    } else if (fColorXformSrcRow) {
273        decodeDst = (uint8_t*) fColorXformSrcRow;
274        swizzleDst = fColorXformSrcRow;
275        decodeDstRowBytes = 0;
276        swizzleDstRowBytes = 0;
277    } else if (fSwizzleSrcRow) {
278        decodeDst = fSwizzleSrcRow;
279        decodeDstRowBytes = 0;
280        dstWidth = fSwizzler->swizzleWidth();
281    }
282
283    for (int y = 0; y < count; y++) {
284        if (!fHeifDecoder->getScanline(decodeDst)) {
285            return y;
286        }
287
288        if (fSwizzler) {
289            fSwizzler->swizzle(swizzleDst, decodeDst);
290        }
291
292        if (this->colorXform()) {
293            this->applyColorXform(dst, swizzleDst, dstWidth);
294            dst = SkTAddOffset<void>(dst, rowBytes);
295        }
296
297        decodeDst = SkTAddOffset<uint8_t>(decodeDst, decodeDstRowBytes);
298        swizzleDst = SkTAddOffset<uint32_t>(swizzleDst, swizzleDstRowBytes);
299    }
300
301    return count;
302}
303
304int SkHeifCodec::onGetFrameCount() {
305    if (!fUseAnimation) {
306        return 1;
307    }
308
309    if (fFrameHolder.size() == 0) {
310        size_t frameCount;
311        HeifFrameInfo frameInfo;
312        if (!fHeifDecoder->getSequenceInfo(&frameInfo, &frameCount)
313                || frameCount <= 1) {
314            fUseAnimation = false;
315            return 1;
316        }
317        fFrameHolder.reserve(frameCount);
318        for (size_t i = 0; i < frameCount; i++) {
319            Frame* frame = fFrameHolder.appendNewFrame();
320            frame->setXYWH(0, 0, frameInfo.mWidth, frameInfo.mHeight);
321            frame->setDisposalMethod(SkCodecAnimation::DisposalMethod::kKeep);
322            // Currently we don't know the duration until the frame is actually
323            // decoded (onGetFrameInfo is also called before frame is decoded).
324            // For now, fill it base on the value reported for the sequence.
325            frame->setDuration(frameInfo.mDurationUs / 1000);
326            frame->setRequiredFrame(SkCodec::kNoFrame);
327            frame->setHasAlpha(false);
328        }
329    }
330
331    return fFrameHolder.size();
332}
333
334const SkFrame* SkHeifCodec::FrameHolder::onGetFrame(int i) const {
335    return static_cast<const SkFrame*>(this->frame(i));
336}
337
338SkHeifCodec::Frame* SkHeifCodec::FrameHolder::appendNewFrame() {
339    const int i = this->size();
340    fFrames.emplace_back(i); // TODO: need to handle frame duration here
341    return &fFrames[i];
342}
343
344const SkHeifCodec::Frame* SkHeifCodec::FrameHolder::frame(int i) const {
345    SkASSERT(i >= 0 && i < this->size());
346    return &fFrames[i];
347}
348
349SkHeifCodec::Frame* SkHeifCodec::FrameHolder::editFrameAt(int i) {
350    SkASSERT(i >= 0 && i < this->size());
351    return &fFrames[i];
352}
353
354bool SkHeifCodec::onGetFrameInfo(int i, FrameInfo* frameInfo) const {
355    if (i >= fFrameHolder.size()) {
356        return false;
357    }
358
359    const Frame* frame = fFrameHolder.frame(i);
360    if (!frame) {
361        return false;
362    }
363
364    if (frameInfo) {
365        frame->fillIn(frameInfo, true);
366    }
367
368    return true;
369}
370
371int SkHeifCodec::onGetRepetitionCount() {
372    return kRepetitionCountInfinite;
373}
374
375/*
376 * Performs the heif decode
377 */
378SkCodec::Result SkHeifCodec::onGetPixels(const SkImageInfo& dstInfo,
379                                         void* dst, size_t dstRowBytes,
380                                         const Options& options,
381                                         int* rowsDecoded) {
382    if (options.fSubset) {
383        // Not supporting subsets on this path for now.
384        // TODO: if the heif has tiles, we can support subset here, but
385        // need to retrieve tile config from metadata retriever first.
386        return kUnimplemented;
387    }
388
389    bool success;
390    if (fUseAnimation) {
391        success = fHeifDecoder->decodeSequence(options.fFrameIndex, &fFrameInfo);
392        fFrameHolder.editFrameAt(options.fFrameIndex)->setDuration(
393                fFrameInfo.mDurationUs / 1000);
394    } else {
395        fHeifDecoder->setDstBuffer((uint8_t *)dst, dstRowBytes, nullptr);
396        success = fHeifDecoder->decode(&fFrameInfo);
397    }
398
399    if (!success) {
400        return kInvalidInput;
401    }
402
403    fSwizzler.reset(nullptr);
404    this->allocateStorage(dstInfo);
405
406    int rows = this->readRows(dstInfo, dst, dstRowBytes, dstInfo.height(), options);
407    if (rows < dstInfo.height()) {
408        *rowsDecoded = rows;
409        return kIncompleteInput;
410    }
411
412    return kSuccess;
413}
414
415void SkHeifCodec::allocateStorage(const SkImageInfo& dstInfo) {
416    int dstWidth = dstInfo.width();
417
418    size_t swizzleBytes = 0;
419    if (fSwizzler) {
420        swizzleBytes = fFrameInfo.mBytesPerPixel * fFrameInfo.mWidth;
421        dstWidth = fSwizzler->swizzleWidth();
422        SkASSERT(!this->colorXform() || SkIsAlign4(swizzleBytes));
423    }
424
425    size_t xformBytes = 0;
426    if (this->colorXform() && (kRGBA_F16_SkColorType == dstInfo.colorType() ||
427                               kRGB_565_SkColorType == dstInfo.colorType())) {
428        xformBytes = dstWidth * sizeof(uint32_t);
429    }
430
431    size_t totalBytes = swizzleBytes + xformBytes;
432    fStorage.reset(totalBytes);
433    if (totalBytes > 0) {
434        fSwizzleSrcRow = (swizzleBytes > 0) ? fStorage.get() : nullptr;
435        fColorXformSrcRow = (xformBytes > 0) ?
436                SkTAddOffset<uint32_t>(fStorage.get(), swizzleBytes) : nullptr;
437    }
438}
439
440void SkHeifCodec::initializeSwizzler(
441        const SkImageInfo& dstInfo, const Options& options) {
442    SkImageInfo swizzlerDstInfo = dstInfo;
443    if (this->colorXform()) {
444        // The color xform will be expecting RGBA 8888 input.
445        swizzlerDstInfo = swizzlerDstInfo.makeColorType(kRGBA_8888_SkColorType);
446    }
447
448    int srcBPP = 4;
449    if (dstInfo.colorType() == kRGB_565_SkColorType && !this->colorXform()) {
450        srcBPP = 2;
451    }
452
453    fSwizzler = SkSwizzler::MakeSimple(srcBPP, swizzlerDstInfo, options);
454    SkASSERT(fSwizzler);
455}
456
457SkSampler* SkHeifCodec::getSampler(bool createIfNecessary) {
458    if (!createIfNecessary || fSwizzler) {
459        SkASSERT(!fSwizzler || (fSwizzleSrcRow && fStorage.get() == fSwizzleSrcRow));
460        return fSwizzler.get();
461    }
462
463    this->initializeSwizzler(this->dstInfo(), this->options());
464    this->allocateStorage(this->dstInfo());
465    return fSwizzler.get();
466}
467
468bool SkHeifCodec::onRewind() {
469    fSwizzler.reset(nullptr);
470    fSwizzleSrcRow = nullptr;
471    fColorXformSrcRow = nullptr;
472    fStorage.reset();
473
474    return true;
475}
476
477SkCodec::Result SkHeifCodec::onStartScanlineDecode(
478        const SkImageInfo& dstInfo, const Options& options) {
479    // TODO: For now, just decode the whole thing even when there is a subset.
480    // If the heif image has tiles, we could potentially do this much faster,
481    // but the tile configuration needs to be retrieved from the metadata.
482    if (!fHeifDecoder->decode(&fFrameInfo)) {
483        return kInvalidInput;
484    }
485
486    if (options.fSubset) {
487        this->initializeSwizzler(dstInfo, options);
488    } else {
489        fSwizzler.reset(nullptr);
490    }
491
492    this->allocateStorage(dstInfo);
493
494    return kSuccess;
495}
496
497int SkHeifCodec::onGetScanlines(void* dst, int count, size_t dstRowBytes) {
498    return this->readRows(this->dstInfo(), dst, dstRowBytes, count, this->options());
499}
500
501bool SkHeifCodec::onSkipScanlines(int count) {
502    return count == (int) fHeifDecoder->skipScanlines(count);
503}
504
505void *SkHeifCodec::heifImplHandle = nullptr;
506HeifDecoder* SkHeifCodec::createHeifDecoder() {
507#if !defined(WIN32) && !defined(_WIN32) && !defined(__WIN32__)
508    if (!heifImplHandle) {
509        heifImplHandle = dlopen(HEIF_IMPL_SO_NAME, RTLD_LAZY);
510    }
511    if (!heifImplHandle) {
512        return new StubHeifDecoder();
513    }
514    typedef HeifDecoder* (*CreateHeifDecoderImplT)();
515    HeifDecoder* decoder = nullptr;
516    CreateHeifDecoderImplT func = (CreateHeifDecoderImplT)dlsym(heifImplHandle, "CreateHeifDecoderImpl");
517    if (func) {
518        decoder = func();
519    }
520    return decoder != nullptr ? decoder : new StubHeifDecoder();
521#else
522    return new StubHeifDecoder();
523#endif
524}
525
526#endif // SK_HAS_HEIF_LIBRARY
527