1/*
2 * Mesa 3-D graphics library
3 *
4 * Copyright (C) 2010-2011 Chia-I Wu <olvaffe@gmail.com>
5 * Copyright (C) 2010-2011 LunarG Inc.
6 *
7 * Based on platform_x11, which has
8 *
9 * Copyright © 2011 Intel Corporation
10 *
11 * Permission is hereby granted, free of charge, to any person obtaining a
12 * copy of this software and associated documentation files (the "Software"),
13 * to deal in the Software without restriction, including without limitation
14 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
15 * and/or sell copies of the Software, and to permit persons to whom the
16 * Software is furnished to do so, subject to the following conditions:
17 *
18 * The above copyright notice and this permission notice shall be included
19 * in all copies or substantial portions of the Software.
20 *
21 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
22 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
23 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
24 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
25 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
26 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
27 * DEALINGS IN THE SOFTWARE.
28 */
29
30#include <cutils/properties.h>
31#include <errno.h>
32#include <dirent.h>
33#include <dlfcn.h>
34#include <fcntl.h>
35#include <xf86drm.h>
36#include <stdbool.h>
37#include <stdio.h>
38#include <sync/sync.h>
39#include <sys/types.h>
40#include <drm-uapi/drm_fourcc.h>
41
42#include "util/compiler.h"
43#include "util/os_file.h"
44#include "util/libsync.h"
45
46#include "loader.h"
47#include "egl_dri2.h"
48#include "platform_android.h"
49
50#ifdef HAVE_DRM_GRALLOC
51#include <gralloc_drm_handle.h>
52#include "gralloc_drm.h"
53#endif /* HAVE_DRM_GRALLOC */
54
55#define ALIGN(val, align)	(((val) + (align) - 1) & ~((align) - 1))
56
57enum chroma_order {
58   YCbCr,
59   YCrCb,
60};
61
62struct droid_yuv_format {
63   /* Lookup keys */
64   int native; /* HAL_PIXEL_FORMAT_ */
65   enum chroma_order chroma_order; /* chroma order is {Cb, Cr} or {Cr, Cb} */
66   int chroma_step; /* Distance in bytes between subsequent chroma pixels. */
67
68   /* Result */
69   int fourcc; /* DRM_FORMAT_ */
70};
71
72/* The following table is used to look up a DRI image FourCC based
73 * on native format and information contained in android_ycbcr struct. */
74static const struct droid_yuv_format droid_yuv_formats[] = {
75   /* Native format, YCrCb, Chroma step, DRI image FourCC */
76   { HAL_PIXEL_FORMAT_YCbCr_420_888, YCbCr, 2, DRM_FORMAT_NV12 },
77   { HAL_PIXEL_FORMAT_YCbCr_420_888, YCbCr, 1, DRM_FORMAT_YUV420 },
78   { HAL_PIXEL_FORMAT_YCbCr_420_888, YCrCb, 1, DRM_FORMAT_YVU420 },
79   { HAL_PIXEL_FORMAT_YV12,          YCrCb, 1, DRM_FORMAT_YVU420 },
80   /* HACK: See droid_create_image_from_prime_fds() and
81    * https://issuetracker.google.com/32077885. */
82   { HAL_PIXEL_FORMAT_IMPLEMENTATION_DEFINED, YCbCr, 2, DRM_FORMAT_NV12 },
83   { HAL_PIXEL_FORMAT_IMPLEMENTATION_DEFINED, YCbCr, 1, DRM_FORMAT_YUV420 },
84   { HAL_PIXEL_FORMAT_IMPLEMENTATION_DEFINED, YCrCb, 1, DRM_FORMAT_YVU420 },
85   { HAL_PIXEL_FORMAT_IMPLEMENTATION_DEFINED, YCrCb, 1, DRM_FORMAT_AYUV },
86   { HAL_PIXEL_FORMAT_IMPLEMENTATION_DEFINED, YCrCb, 1, DRM_FORMAT_XYUV8888 },
87};
88
89static int
90get_fourcc_yuv(int native, enum chroma_order chroma_order, int chroma_step)
91{
92   for (int i = 0; i < ARRAY_SIZE(droid_yuv_formats); ++i)
93      if (droid_yuv_formats[i].native == native &&
94          droid_yuv_formats[i].chroma_order == chroma_order &&
95          droid_yuv_formats[i].chroma_step == chroma_step)
96         return droid_yuv_formats[i].fourcc;
97
98   return -1;
99}
100
101static bool
102is_yuv(int native)
103{
104   for (int i = 0; i < ARRAY_SIZE(droid_yuv_formats); ++i)
105      if (droid_yuv_formats[i].native == native)
106         return true;
107
108   return false;
109}
110
111static int
112get_format_bpp(int native)
113{
114   int bpp;
115
116   switch (native) {
117   case HAL_PIXEL_FORMAT_RGBA_FP16:
118      bpp = 8;
119      break;
120   case HAL_PIXEL_FORMAT_RGBA_8888:
121   case HAL_PIXEL_FORMAT_IMPLEMENTATION_DEFINED:
122      /*
123       * HACK: Hardcode this to RGBX_8888 as per cros_gralloc hack.
124       * TODO: Remove this once https://issuetracker.google.com/32077885 is fixed.
125       */
126   case HAL_PIXEL_FORMAT_RGBX_8888:
127   case HAL_PIXEL_FORMAT_BGRA_8888:
128   case HAL_PIXEL_FORMAT_RGBA_1010102:
129      bpp = 4;
130      break;
131   case HAL_PIXEL_FORMAT_RGB_565:
132      bpp = 2;
133      break;
134   default:
135      bpp = 0;
136      break;
137   }
138
139   return bpp;
140}
141
142/* createImageFromFds requires fourcc format */
143static int get_fourcc(int native)
144{
145   switch (native) {
146   case HAL_PIXEL_FORMAT_RGB_565:   return DRM_FORMAT_RGB565;
147   case HAL_PIXEL_FORMAT_BGRA_8888: return DRM_FORMAT_ARGB8888;
148   case HAL_PIXEL_FORMAT_RGBA_8888: return DRM_FORMAT_ABGR8888;
149   case HAL_PIXEL_FORMAT_IMPLEMENTATION_DEFINED:
150      /*
151       * HACK: Hardcode this to RGBX_8888 as per cros_gralloc hack.
152       * TODO: Remove this once https://issuetracker.google.com/32077885 is fixed.
153       */
154   case HAL_PIXEL_FORMAT_RGBX_8888: return DRM_FORMAT_XBGR8888;
155   case HAL_PIXEL_FORMAT_RGBA_FP16: return DRM_FORMAT_ABGR16161616F;
156   case HAL_PIXEL_FORMAT_RGBA_1010102: return DRM_FORMAT_ABGR2101010;
157   default:
158      _eglLog(_EGL_WARNING, "unsupported native buffer format 0x%x", native);
159   }
160   return -1;
161}
162
163/* returns # of fds, and by reference the actual fds */
164static unsigned
165get_native_buffer_fds(struct ANativeWindowBuffer *buf, int fds[3])
166{
167   native_handle_t *handle = (native_handle_t *)buf->handle;
168
169   if (!handle)
170      return 0;
171
172   /*
173    * Various gralloc implementations exist, but the dma-buf fd tends
174    * to be first. Access it directly to avoid a dependency on specific
175    * gralloc versions.
176    */
177   for (int i = 0; i < handle->numFds; i++)
178      fds[i] = handle->data[i];
179
180   return handle->numFds;
181}
182
183#ifdef HAVE_DRM_GRALLOC
184static int
185get_native_buffer_name(struct ANativeWindowBuffer *buf)
186{
187   return gralloc_drm_get_gem_handle(buf->handle);
188}
189#endif /* HAVE_DRM_GRALLOC */
190
191static int
192get_yuv_buffer_info(struct dri2_egl_display *dri2_dpy,
193                    struct ANativeWindowBuffer *buf,
194                    struct buffer_info *out_buf_info)
195{
196   struct android_ycbcr ycbcr;
197   enum chroma_order chroma_order;
198   int drm_fourcc = 0;
199   int num_fds = 0;
200   int fds[3];
201   int ret;
202
203   num_fds = get_native_buffer_fds(buf, fds);
204   if (num_fds == 0)
205      return -EINVAL;
206
207   if (!dri2_dpy->gralloc->lock_ycbcr) {
208      _eglLog(_EGL_WARNING, "Gralloc does not support lock_ycbcr");
209      return -EINVAL;
210   }
211
212   memset(&ycbcr, 0, sizeof(ycbcr));
213   ret = dri2_dpy->gralloc->lock_ycbcr(dri2_dpy->gralloc, buf->handle,
214                                       0, 0, 0, 0, 0, &ycbcr);
215   if (ret) {
216      /* HACK: See native_window_buffer_get_buffer_info() and
217       * https://issuetracker.google.com/32077885.*/
218      if (buf->format == HAL_PIXEL_FORMAT_IMPLEMENTATION_DEFINED)
219         return -EAGAIN;
220
221      _eglLog(_EGL_WARNING, "gralloc->lock_ycbcr failed: %d", ret);
222      return -EINVAL;
223   }
224   dri2_dpy->gralloc->unlock(dri2_dpy->gralloc, buf->handle);
225
226   chroma_order = ((size_t)ycbcr.cr < (size_t)ycbcr.cb) ? YCrCb : YCbCr;
227
228   /* .chroma_step is the byte distance between the same chroma channel
229    * values of subsequent pixels, assumed to be the same for Cb and Cr. */
230   drm_fourcc = get_fourcc_yuv(buf->format, chroma_order, ycbcr.chroma_step);
231   if (drm_fourcc == -1) {
232      _eglLog(_EGL_WARNING, "unsupported YUV format, native = %x, chroma_order = %s, chroma_step = %d",
233              buf->format, chroma_order == YCbCr ? "YCbCr" : "YCrCb", ycbcr.chroma_step);
234      return -EINVAL;
235   }
236
237   *out_buf_info = (struct buffer_info){
238      .width = buf->width,
239      .height = buf->height,
240      .drm_fourcc = drm_fourcc,
241      .num_planes = ycbcr.chroma_step == 2 ? 2 : 3,
242      .fds = { -1, -1, -1, -1 },
243      .modifier = DRM_FORMAT_MOD_INVALID,
244      .yuv_color_space = EGL_ITU_REC601_EXT,
245      .sample_range = EGL_YUV_NARROW_RANGE_EXT,
246      .horizontal_siting = EGL_YUV_CHROMA_SITING_0_EXT,
247      .vertical_siting = EGL_YUV_CHROMA_SITING_0_EXT,
248   };
249
250   /* When lock_ycbcr's usage argument contains no SW_READ/WRITE flags
251    * it will return the .y/.cb/.cr pointers based on a NULL pointer,
252    * so they can be interpreted as offsets. */
253   out_buf_info->offsets[0] = (size_t)ycbcr.y;
254   /* We assume here that all the planes are located in one DMA-buf. */
255   if (chroma_order == YCrCb) {
256      out_buf_info->offsets[1] = (size_t)ycbcr.cr;
257      out_buf_info->offsets[2] = (size_t)ycbcr.cb;
258   } else {
259      out_buf_info->offsets[1] = (size_t)ycbcr.cb;
260      out_buf_info->offsets[2] = (size_t)ycbcr.cr;
261   }
262
263   /* .ystride is the line length (in bytes) of the Y plane,
264    * .cstride is the line length (in bytes) of any of the remaining
265    * Cb/Cr/CbCr planes, assumed to be the same for Cb and Cr for fully
266    * planar formats. */
267   out_buf_info->pitches[0] = ycbcr.ystride;
268   out_buf_info->pitches[1] = out_buf_info->pitches[2] = ycbcr.cstride;
269
270   /*
271    * Since this is EGL_NATIVE_BUFFER_ANDROID don't assume that
272    * the single-fd case cannot happen.  So handle eithe single
273    * fd or fd-per-plane case:
274    */
275   if (num_fds == 1) {
276      out_buf_info->fds[1] = out_buf_info->fds[0] = fds[0];
277      if (out_buf_info->num_planes == 3)
278         out_buf_info->fds[2] = fds[0];
279   } else {
280      assert(num_fds == out_buf_info->num_planes);
281      out_buf_info->fds[0] = fds[0];
282      out_buf_info->fds[1] = fds[1];
283      out_buf_info->fds[2] = fds[2];
284   }
285
286   return 0;
287}
288
289static int
290native_window_buffer_get_buffer_info(struct dri2_egl_display *dri2_dpy,
291                                     struct ANativeWindowBuffer *buf,
292                                     struct buffer_info *out_buf_info)
293{
294   int num_planes = 0;
295   int drm_fourcc = 0;
296   int pitch = 0;
297   int fds[3];
298
299   if (is_yuv(buf->format)) {
300      int ret = get_yuv_buffer_info(dri2_dpy, buf, out_buf_info);
301      /*
302       * HACK: https://issuetracker.google.com/32077885
303       * There is no API available to properly query the IMPLEMENTATION_DEFINED
304       * format. As a workaround we rely here on gralloc allocating either
305       * an arbitrary YCbCr 4:2:0 or RGBX_8888, with the latter being recognized
306       * by lock_ycbcr failing.
307       */
308      if (ret != -EAGAIN)
309         return ret;
310   }
311
312   /*
313    * Non-YUV formats could *also* have multiple planes, such as ancillary
314    * color compression state buffer, but the rest of the code isn't ready
315    * yet to deal with modifiers:
316    */
317   num_planes = get_native_buffer_fds(buf, fds);
318   if (num_planes == 0)
319      return -EINVAL;
320
321   assert(num_planes == 1);
322
323   drm_fourcc = get_fourcc(buf->format);
324   if (drm_fourcc == -1) {
325      _eglError(EGL_BAD_PARAMETER, "eglCreateEGLImageKHR");
326      return -EINVAL;
327   }
328
329   pitch = buf->stride * get_format_bpp(buf->format);
330   if (pitch == 0) {
331      _eglError(EGL_BAD_PARAMETER, "eglCreateEGLImageKHR");
332      return -EINVAL;
333   }
334
335   *out_buf_info = (struct buffer_info){
336      .width = buf->width,
337      .height = buf->height,
338      .drm_fourcc = drm_fourcc,
339      .num_planes = num_planes,
340      .fds = { fds[0], -1, -1, -1 },
341      .modifier = DRM_FORMAT_MOD_INVALID,
342      .offsets = { 0, 0, 0, 0 },
343      .pitches = { pitch, 0, 0, 0 },
344      .yuv_color_space = EGL_ITU_REC601_EXT,
345      .sample_range = EGL_YUV_NARROW_RANGE_EXT,
346      .horizontal_siting = EGL_YUV_CHROMA_SITING_0_EXT,
347      .vertical_siting = EGL_YUV_CHROMA_SITING_0_EXT,
348   };
349
350   return 0;
351}
352
353/* More recent CrOS gralloc has a perform op that fills out the struct below
354 * with canonical information about the buffer and its modifier, planes,
355 * offsets and strides.  If we have this, we can skip straight to
356 * createImageFromDmaBufs2() and avoid all the guessing and recalculations.
357 * This also gives us the modifier and plane offsets/strides for multiplanar
358 * compressed buffers (eg Intel CCS buffers) in order to make that work in Android.
359 */
360
361static const char cros_gralloc_module_name[] = "CrOS Gralloc";
362
363#define CROS_GRALLOC_DRM_GET_BUFFER_INFO 4
364#define CROS_GRALLOC_DRM_GET_USAGE 5
365#define CROS_GRALLOC_DRM_GET_USAGE_FRONT_RENDERING_BIT 0x1
366
367struct cros_gralloc0_buffer_info {
368   uint32_t drm_fourcc;
369   int num_fds;
370   int fds[4];
371   uint64_t modifier;
372   int offset[4];
373   int stride[4];
374};
375
376static int
377cros_get_buffer_info(struct dri2_egl_display *dri2_dpy,
378                     struct ANativeWindowBuffer *buf,
379                     struct buffer_info *out_buf_info)
380{
381   struct cros_gralloc0_buffer_info info;
382
383   if (strcmp(dri2_dpy->gralloc->common.name, cros_gralloc_module_name) == 0 &&
384       dri2_dpy->gralloc->perform &&
385       dri2_dpy->gralloc->perform(dri2_dpy->gralloc,
386                                  CROS_GRALLOC_DRM_GET_BUFFER_INFO,
387                                  buf->handle, &info) == 0) {
388      *out_buf_info = (struct buffer_info){
389         .width = buf->width,
390         .height = buf->height,
391         .drm_fourcc = info.drm_fourcc,
392         .num_planes = info.num_fds,
393         .fds = { -1, -1, -1, -1 },
394         .modifier = info.modifier,
395         .yuv_color_space = EGL_ITU_REC601_EXT,
396         .sample_range = EGL_YUV_NARROW_RANGE_EXT,
397         .horizontal_siting = EGL_YUV_CHROMA_SITING_0_EXT,
398         .vertical_siting = EGL_YUV_CHROMA_SITING_0_EXT,
399      };
400      for (int i = 0; i < out_buf_info->num_planes; i++) {
401         out_buf_info->fds[i] = info.fds[i];
402         out_buf_info->offsets[i] = info.offset[i];
403         out_buf_info->pitches[i] = info.stride[i];
404      }
405
406      return 0;
407   }
408
409   return -EINVAL;
410}
411
412static __DRIimage *
413droid_create_image_from_buffer_info(struct dri2_egl_display *dri2_dpy,
414                                    struct buffer_info *buf_info,
415                                    void *priv)
416{
417   unsigned error;
418
419   if (dri2_dpy->image->base.version >= 15 &&
420       dri2_dpy->image->createImageFromDmaBufs2 != NULL) {
421      return dri2_dpy->image->createImageFromDmaBufs2(
422         dri2_dpy->dri_screen, buf_info->width, buf_info->height,
423         buf_info->drm_fourcc, buf_info->modifier, buf_info->fds,
424         buf_info->num_planes, buf_info->pitches, buf_info->offsets,
425         buf_info->yuv_color_space, buf_info->sample_range,
426         buf_info->horizontal_siting, buf_info->vertical_siting, &error,
427         priv);
428   }
429
430   return dri2_dpy->image->createImageFromDmaBufs(
431      dri2_dpy->dri_screen, buf_info->width, buf_info->height,
432      buf_info->drm_fourcc, buf_info->fds, buf_info->num_planes,
433      buf_info->pitches, buf_info->offsets, buf_info->yuv_color_space,
434      buf_info->sample_range, buf_info->horizontal_siting,
435      buf_info->vertical_siting, &error, priv);
436}
437
438static __DRIimage *
439droid_create_image_from_native_buffer(_EGLDisplay *disp,
440                                      struct ANativeWindowBuffer *buf,
441                                      void *priv)
442{
443   struct dri2_egl_display *dri2_dpy = dri2_egl_display(disp);
444   struct buffer_info buf_info;
445   __DRIimage *img = NULL;
446
447   /* If dri driver is gallium virgl, real modifier info queried back from
448    * CrOS info (and potentially mapper metadata if integrated later) cannot
449    * get resolved and the buffer import will fail. Thus the fallback behavior
450    * is preserved down to native_window_buffer_get_buffer_info() so that the
451    * buffer can be imported without modifier info as a last resort.
452    */
453   if (!img && !mapper_metadata_get_buffer_info(buf, &buf_info))
454      img = droid_create_image_from_buffer_info(dri2_dpy, &buf_info, priv);
455
456   if (!img && !cros_get_buffer_info(dri2_dpy, buf, &buf_info))
457      img = droid_create_image_from_buffer_info(dri2_dpy, &buf_info, priv);
458
459   if (!img && !native_window_buffer_get_buffer_info(dri2_dpy, buf, &buf_info))
460      img = droid_create_image_from_buffer_info(dri2_dpy, &buf_info, priv);
461
462   return img;
463}
464
465static void
466handle_in_fence_fd(struct dri2_egl_surface *dri2_surf, __DRIimage *img)
467{
468   _EGLDisplay *disp = dri2_surf->base.Resource.Display;
469   struct dri2_egl_display *dri2_dpy = dri2_egl_display(disp);
470
471   if (dri2_surf->in_fence_fd < 0)
472      return;
473
474   validate_fence_fd(dri2_surf->in_fence_fd);
475
476   if (dri2_dpy->image->base.version >= 21 &&
477       dri2_dpy->image->setInFenceFd != NULL) {
478      dri2_dpy->image->setInFenceFd(img, dri2_surf->in_fence_fd);
479   } else {
480      sync_wait(dri2_surf->in_fence_fd, -1);
481   }
482}
483
484static void
485close_in_fence_fd(struct dri2_egl_surface *dri2_surf)
486{
487   validate_fence_fd(dri2_surf->in_fence_fd);
488   if (dri2_surf->in_fence_fd >= 0)
489      close(dri2_surf->in_fence_fd);
490   dri2_surf->in_fence_fd = -1;
491}
492
493static EGLBoolean
494droid_window_dequeue_buffer(struct dri2_egl_surface *dri2_surf)
495{
496   int fence_fd;
497
498   if (ANativeWindow_dequeueBuffer(dri2_surf->window, &dri2_surf->buffer,
499                                   &fence_fd))
500      return EGL_FALSE;
501
502   close_in_fence_fd(dri2_surf);
503
504   validate_fence_fd(fence_fd);
505
506   dri2_surf->in_fence_fd = fence_fd;
507
508   /* Record all the buffers created by ANativeWindow and update back buffer
509    * for updating buffer's age in swap_buffers.
510    */
511   EGLBoolean updated = EGL_FALSE;
512   for (int i = 0; i < dri2_surf->color_buffers_count; i++) {
513      if (!dri2_surf->color_buffers[i].buffer) {
514         dri2_surf->color_buffers[i].buffer = dri2_surf->buffer;
515      }
516      if (dri2_surf->color_buffers[i].buffer == dri2_surf->buffer) {
517         dri2_surf->back = &dri2_surf->color_buffers[i];
518         updated = EGL_TRUE;
519         break;
520      }
521   }
522
523   if (!updated) {
524      /* In case of all the buffers were recreated by ANativeWindow, reset
525       * the color_buffers
526       */
527      for (int i = 0; i < dri2_surf->color_buffers_count; i++) {
528         dri2_surf->color_buffers[i].buffer = NULL;
529         dri2_surf->color_buffers[i].age = 0;
530      }
531      dri2_surf->color_buffers[0].buffer = dri2_surf->buffer;
532      dri2_surf->back = &dri2_surf->color_buffers[0];
533   }
534
535   return EGL_TRUE;
536}
537
538static EGLBoolean
539droid_window_enqueue_buffer(_EGLDisplay *disp, struct dri2_egl_surface *dri2_surf)
540{
541   struct dri2_egl_display *dri2_dpy = dri2_egl_display(disp);
542
543   /* To avoid blocking other EGL calls, release the display mutex before
544    * we enter droid_window_enqueue_buffer() and re-acquire the mutex upon
545    * return.
546    */
547   mtx_unlock(&disp->Mutex);
548
549   /* Queue the buffer with stored out fence fd. The ANativeWindow or buffer
550    * consumer may choose to wait for the fence to signal before accessing
551    * it. If fence fd value is -1, buffer can be accessed by consumer
552    * immediately. Consumer or application shouldn't rely on timestamp
553    * associated with fence if the fence fd is -1.
554    *
555    * Ownership of fd is transferred to consumer after queueBuffer and the
556    * consumer is responsible for closing it. Caller must not use the fd
557    * after passing it to queueBuffer.
558    */
559   int fence_fd = dri2_surf->out_fence_fd;
560   dri2_surf->out_fence_fd = -1;
561   ANativeWindow_queueBuffer(dri2_surf->window, dri2_surf->buffer, fence_fd);
562
563   dri2_surf->buffer = NULL;
564   dri2_surf->back = NULL;
565
566   mtx_lock(&disp->Mutex);
567
568   if (dri2_surf->dri_image_back) {
569      dri2_dpy->image->destroyImage(dri2_surf->dri_image_back);
570      dri2_surf->dri_image_back = NULL;
571   }
572
573   return EGL_TRUE;
574}
575
576static void
577droid_window_cancel_buffer(struct dri2_egl_surface *dri2_surf)
578{
579   int ret;
580   int fence_fd = dri2_surf->out_fence_fd;
581
582   dri2_surf->out_fence_fd = -1;
583   ret = ANativeWindow_cancelBuffer(dri2_surf->window, dri2_surf->buffer,
584                                    fence_fd);
585   dri2_surf->buffer = NULL;
586   if (ret < 0) {
587      _eglLog(_EGL_WARNING, "ANativeWindow_cancelBuffer failed");
588      dri2_surf->base.Lost = EGL_TRUE;
589   }
590
591   close_in_fence_fd(dri2_surf);
592}
593
594static bool
595droid_set_shared_buffer_mode(_EGLDisplay *disp, _EGLSurface *surf, bool mode)
596{
597#if ANDROID_API_LEVEL >= 24
598   struct dri2_egl_display *dri2_dpy = dri2_egl_display(disp);
599   struct dri2_egl_surface *dri2_surf = dri2_egl_surface(surf);
600   struct ANativeWindow *window = dri2_surf->window;
601
602   assert(surf->Type == EGL_WINDOW_BIT);
603   assert(_eglSurfaceHasMutableRenderBuffer(&dri2_surf->base));
604
605   _eglLog(_EGL_DEBUG, "%s: mode=%d", __func__, mode);
606
607   if (ANativeWindow_setSharedBufferMode(window, mode)) {
608      _eglLog(_EGL_WARNING, "failed ANativeWindow_setSharedBufferMode"
609              "(window=%p, mode=%d)", window, mode);
610      return false;
611   }
612
613   if (mode)
614      dri2_surf->gralloc_usage |= dri2_dpy->front_rendering_usage;
615   else
616      dri2_surf->gralloc_usage &= ~dri2_dpy->front_rendering_usage;
617
618   if (ANativeWindow_setUsage(window, dri2_surf->gralloc_usage)) {
619      _eglLog(_EGL_WARNING,
620              "failed ANativeWindow_setUsage(window=%p, usage=%u)", window,
621              dri2_surf->gralloc_usage);
622      return false;
623   }
624
625   return true;
626#else
627   _eglLog(_EGL_FATAL, "%s:%d: internal error: unreachable", __FILE__, __LINE__);
628   return false;
629#endif
630}
631
632static _EGLSurface *
633droid_create_surface(_EGLDisplay *disp, EGLint type, _EGLConfig *conf,
634                     void *native_window, const EGLint *attrib_list)
635{
636   struct dri2_egl_display *dri2_dpy = dri2_egl_display(disp);
637   struct dri2_egl_config *dri2_conf = dri2_egl_config(conf);
638   struct dri2_egl_surface *dri2_surf;
639   struct ANativeWindow *window = native_window;
640   const __DRIconfig *config;
641
642   dri2_surf = calloc(1, sizeof *dri2_surf);
643   if (!dri2_surf) {
644      _eglError(EGL_BAD_ALLOC, "droid_create_surface");
645      return NULL;
646   }
647
648   dri2_surf->in_fence_fd = -1;
649
650   if (!dri2_init_surface(&dri2_surf->base, disp, type, conf, attrib_list,
651                          true, native_window))
652      goto cleanup_surface;
653
654   if (type == EGL_WINDOW_BIT) {
655      int format;
656      int buffer_count;
657      int min_undequeued_buffers;
658
659      format = ANativeWindow_getFormat(window);
660      if (format < 0) {
661         _eglError(EGL_BAD_NATIVE_WINDOW, "droid_create_surface");
662         goto cleanup_surface;
663      }
664
665      /* Query ANativeWindow for MIN_UNDEQUEUED_BUFFER, minimum amount
666       * of undequeued buffers.
667       */
668      if (ANativeWindow_query(window,
669                              ANATIVEWINDOW_QUERY_MIN_UNDEQUEUED_BUFFERS,
670                              &min_undequeued_buffers)) {
671         _eglError(EGL_BAD_NATIVE_WINDOW, "droid_create_surface");
672         goto cleanup_surface;
673      }
674
675      /* Required buffer caching slots. */
676      buffer_count = min_undequeued_buffers + 2;
677
678      dri2_surf->color_buffers = calloc(buffer_count,
679                                        sizeof(*dri2_surf->color_buffers));
680      if (!dri2_surf->color_buffers) {
681         _eglError(EGL_BAD_ALLOC, "droid_create_surface");
682         goto cleanup_surface;
683      }
684      dri2_surf->color_buffers_count = buffer_count;
685
686      if (format != dri2_conf->base.NativeVisualID) {
687         _eglLog(_EGL_WARNING, "Native format mismatch: 0x%x != 0x%x",
688               format, dri2_conf->base.NativeVisualID);
689      }
690
691      ANativeWindow_query(window, ANATIVEWINDOW_QUERY_DEFAULT_WIDTH,
692                          &dri2_surf->base.Width);
693      ANativeWindow_query(window, ANATIVEWINDOW_QUERY_DEFAULT_HEIGHT,
694                          &dri2_surf->base.Height);
695
696      dri2_surf->gralloc_usage =
697         strcmp(dri2_dpy->driver_name, "kms_swrast") == 0
698            ? GRALLOC_USAGE_SW_READ_OFTEN | GRALLOC_USAGE_SW_WRITE_OFTEN
699            : GRALLOC_USAGE_HW_RENDER;
700
701      if (dri2_surf->base.ActiveRenderBuffer == EGL_SINGLE_BUFFER)
702         dri2_surf->gralloc_usage |= dri2_dpy->front_rendering_usage;
703
704      if (ANativeWindow_setUsage(window, dri2_surf->gralloc_usage)) {
705         _eglError(EGL_BAD_NATIVE_WINDOW, "droid_create_surface");
706         goto cleanup_surface;
707      }
708   }
709
710   config = dri2_get_dri_config(dri2_conf, type,
711                                dri2_surf->base.GLColorspace);
712   if (!config) {
713      _eglError(EGL_BAD_MATCH, "Unsupported surfacetype/colorspace configuration");
714      goto cleanup_surface;
715   }
716
717   if (!dri2_create_drawable(dri2_dpy, config, dri2_surf, dri2_surf))
718      goto cleanup_surface;
719
720   if (window) {
721      ANativeWindow_acquire(window);
722      dri2_surf->window = window;
723   }
724
725   return &dri2_surf->base;
726
727cleanup_surface:
728   if (dri2_surf->color_buffers_count)
729      free(dri2_surf->color_buffers);
730   free(dri2_surf);
731
732   return NULL;
733}
734
735static _EGLSurface *
736droid_create_window_surface(_EGLDisplay *disp, _EGLConfig *conf,
737                            void *native_window, const EGLint *attrib_list)
738{
739   return droid_create_surface(disp, EGL_WINDOW_BIT, conf,
740                               native_window, attrib_list);
741}
742
743static _EGLSurface *
744droid_create_pbuffer_surface(_EGLDisplay *disp, _EGLConfig *conf,
745                             const EGLint *attrib_list)
746{
747   return droid_create_surface(disp, EGL_PBUFFER_BIT, conf,
748                               NULL, attrib_list);
749}
750
751static EGLBoolean
752droid_destroy_surface(_EGLDisplay *disp, _EGLSurface *surf)
753{
754   struct dri2_egl_display *dri2_dpy = dri2_egl_display(disp);
755   struct dri2_egl_surface *dri2_surf = dri2_egl_surface(surf);
756
757   dri2_egl_surface_free_local_buffers(dri2_surf);
758
759   if (dri2_surf->base.Type == EGL_WINDOW_BIT) {
760      if (dri2_surf->buffer)
761         droid_window_cancel_buffer(dri2_surf);
762
763      ANativeWindow_release(dri2_surf->window);
764   }
765
766   if (dri2_surf->dri_image_back) {
767      _eglLog(_EGL_DEBUG, "%s : %d : destroy dri_image_back", __func__, __LINE__);
768      dri2_dpy->image->destroyImage(dri2_surf->dri_image_back);
769      dri2_surf->dri_image_back = NULL;
770   }
771
772   if (dri2_surf->dri_image_front) {
773      _eglLog(_EGL_DEBUG, "%s : %d : destroy dri_image_front", __func__, __LINE__);
774      dri2_dpy->image->destroyImage(dri2_surf->dri_image_front);
775      dri2_surf->dri_image_front = NULL;
776   }
777
778   dri2_dpy->core->destroyDrawable(dri2_surf->dri_drawable);
779
780   close_in_fence_fd(dri2_surf);
781   dri2_fini_surface(surf);
782   free(dri2_surf->color_buffers);
783   free(dri2_surf);
784
785   return EGL_TRUE;
786}
787
788static EGLBoolean
789droid_swap_interval(_EGLDisplay *disp, _EGLSurface *surf, EGLint interval)
790{
791   struct dri2_egl_surface *dri2_surf = dri2_egl_surface(surf);
792   struct ANativeWindow *window = dri2_surf->window;
793
794   if (ANativeWindow_setSwapInterval(window, interval))
795      return EGL_FALSE;
796
797   surf->SwapInterval = interval;
798   return EGL_TRUE;
799}
800
801static int
802update_buffers(struct dri2_egl_surface *dri2_surf)
803{
804   if (dri2_surf->base.Lost)
805      return -1;
806
807   if (dri2_surf->base.Type != EGL_WINDOW_BIT)
808      return 0;
809
810   /* try to dequeue the next back buffer */
811   if (!dri2_surf->buffer && !droid_window_dequeue_buffer(dri2_surf)) {
812      _eglLog(_EGL_WARNING, "Could not dequeue buffer from native window");
813      dri2_surf->base.Lost = EGL_TRUE;
814      return -1;
815   }
816
817   /* free outdated buffers and update the surface size */
818   if (dri2_surf->base.Width != dri2_surf->buffer->width ||
819       dri2_surf->base.Height != dri2_surf->buffer->height) {
820      dri2_egl_surface_free_local_buffers(dri2_surf);
821      dri2_surf->base.Width = dri2_surf->buffer->width;
822      dri2_surf->base.Height = dri2_surf->buffer->height;
823   }
824
825   return 0;
826}
827
828static int
829get_front_bo(struct dri2_egl_surface *dri2_surf, unsigned int format)
830{
831   struct dri2_egl_display *dri2_dpy =
832      dri2_egl_display(dri2_surf->base.Resource.Display);
833
834   if (dri2_surf->dri_image_front)
835      return 0;
836
837   if (dri2_surf->base.Type == EGL_WINDOW_BIT) {
838      /* According current EGL spec, front buffer rendering
839       * for window surface is not supported now.
840       * and mesa doesn't have the implementation of this case.
841       * Add warning message, but not treat it as error.
842       */
843      _eglLog(_EGL_DEBUG, "DRI driver requested unsupported front buffer for window surface");
844   } else if (dri2_surf->base.Type == EGL_PBUFFER_BIT) {
845      dri2_surf->dri_image_front =
846          dri2_dpy->image->createImage(dri2_dpy->dri_screen,
847                                              dri2_surf->base.Width,
848                                              dri2_surf->base.Height,
849                                              format,
850                                              0,
851                                              NULL);
852      if (!dri2_surf->dri_image_front) {
853         _eglLog(_EGL_WARNING, "dri2_image_front allocation failed");
854         return -1;
855      }
856   }
857
858   return 0;
859}
860
861static int
862get_back_bo(struct dri2_egl_surface *dri2_surf)
863{
864   _EGLDisplay *disp = dri2_surf->base.Resource.Display;
865
866   if (dri2_surf->dri_image_back)
867      return 0;
868
869   if (dri2_surf->base.Type == EGL_WINDOW_BIT) {
870      if (!dri2_surf->buffer) {
871         _eglLog(_EGL_WARNING, "Could not get native buffer");
872         return -1;
873      }
874
875      dri2_surf->dri_image_back =
876         droid_create_image_from_native_buffer(disp, dri2_surf->buffer, NULL);
877      if (!dri2_surf->dri_image_back) {
878         _eglLog(_EGL_WARNING, "failed to create DRI image from FD");
879         return -1;
880      }
881
882      handle_in_fence_fd(dri2_surf, dri2_surf->dri_image_back);
883
884   } else if (dri2_surf->base.Type == EGL_PBUFFER_BIT) {
885      /* The EGL 1.5 spec states that pbuffers are single-buffered. Specifically,
886       * the spec states that they have a back buffer but no front buffer, in
887       * contrast to pixmaps, which have a front buffer but no back buffer.
888       *
889       * Single-buffered surfaces with no front buffer confuse Mesa; so we deviate
890       * from the spec, following the precedent of Mesa's EGL X11 platform. The
891       * X11 platform correctly assigns pbuffers to single-buffered configs, but
892       * assigns the pbuffer a front buffer instead of a back buffer.
893       *
894       * Pbuffers in the X11 platform mostly work today, so let's just copy its
895       * behavior instead of trying to fix (and hence potentially breaking) the
896       * world.
897       */
898      _eglLog(_EGL_DEBUG, "DRI driver requested unsupported back buffer for pbuffer surface");
899   }
900
901   return 0;
902}
903
904/* Some drivers will pass multiple bits in buffer_mask.
905 * For such case, will go through all the bits, and
906 * will not return error when unsupported buffer is requested, only
907 * return error when the allocation for supported buffer failed.
908 */
909static int
910droid_image_get_buffers(__DRIdrawable *driDrawable,
911                  unsigned int format,
912                  uint32_t *stamp,
913                  void *loaderPrivate,
914                  uint32_t buffer_mask,
915                  struct __DRIimageList *images)
916{
917   struct dri2_egl_surface *dri2_surf = loaderPrivate;
918
919   images->image_mask = 0;
920   images->front = NULL;
921   images->back = NULL;
922
923   if (update_buffers(dri2_surf) < 0)
924      return 0;
925
926   if (_eglSurfaceInSharedBufferMode(&dri2_surf->base)) {
927      if (get_back_bo(dri2_surf) < 0)
928         return 0;
929
930      /* We have dri_image_back because this is a window surface and
931       * get_back_bo() succeeded.
932       */
933      assert(dri2_surf->dri_image_back);
934      images->back = dri2_surf->dri_image_back;
935      images->image_mask |= __DRI_IMAGE_BUFFER_SHARED;
936
937      /* There exists no accompanying back nor front buffer. */
938      return 1;
939   }
940
941   if (buffer_mask & __DRI_IMAGE_BUFFER_FRONT) {
942      if (get_front_bo(dri2_surf, format) < 0)
943         return 0;
944
945      if (dri2_surf->dri_image_front) {
946         images->front = dri2_surf->dri_image_front;
947         images->image_mask |= __DRI_IMAGE_BUFFER_FRONT;
948      }
949   }
950
951   if (buffer_mask & __DRI_IMAGE_BUFFER_BACK) {
952      if (get_back_bo(dri2_surf) < 0)
953         return 0;
954
955      if (dri2_surf->dri_image_back) {
956         images->back = dri2_surf->dri_image_back;
957         images->image_mask |= __DRI_IMAGE_BUFFER_BACK;
958      }
959   }
960
961   return 1;
962}
963
964static EGLint
965droid_query_buffer_age(_EGLDisplay *disp, _EGLSurface *surface)
966{
967   struct dri2_egl_surface *dri2_surf = dri2_egl_surface(surface);
968
969   if (update_buffers(dri2_surf) < 0) {
970      _eglError(EGL_BAD_ALLOC, "droid_query_buffer_age");
971      return -1;
972   }
973
974   return dri2_surf->back ? dri2_surf->back->age : 0;
975}
976
977static EGLBoolean
978droid_swap_buffers(_EGLDisplay *disp, _EGLSurface *draw)
979{
980   struct dri2_egl_display *dri2_dpy = dri2_egl_display(disp);
981   struct dri2_egl_surface *dri2_surf = dri2_egl_surface(draw);
982   const bool has_mutable_rb = _eglSurfaceHasMutableRenderBuffer(draw);
983
984   /* From the EGL_KHR_mutable_render_buffer spec (v12):
985    *
986    *    If surface is a single-buffered window, pixmap, or pbuffer surface
987    *    for which there is no pending change to the EGL_RENDER_BUFFER
988    *    attribute, eglSwapBuffers has no effect.
989    */
990   if (has_mutable_rb &&
991       draw->RequestedRenderBuffer == EGL_SINGLE_BUFFER &&
992       draw->ActiveRenderBuffer == EGL_SINGLE_BUFFER) {
993      _eglLog(_EGL_DEBUG, "%s: remain in shared buffer mode", __func__);
994      return EGL_TRUE;
995   }
996
997   for (int i = 0; i < dri2_surf->color_buffers_count; i++) {
998      if (dri2_surf->color_buffers[i].age > 0)
999         dri2_surf->color_buffers[i].age++;
1000   }
1001
1002   /* "XXX: we don't use get_back_bo() since it causes regressions in
1003    * several dEQP tests.
1004    */
1005   if (dri2_surf->back)
1006      dri2_surf->back->age = 1;
1007
1008   dri2_flush_drawable_for_swapbuffers(disp, draw);
1009
1010   /* dri2_surf->buffer can be null even when no error has occured. For
1011    * example, if the user has called no GL rendering commands since the
1012    * previous eglSwapBuffers, then the driver may have not triggered
1013    * a callback to ANativeWindow_dequeueBuffer, in which case
1014    * dri2_surf->buffer remains null.
1015    */
1016   if (dri2_surf->buffer)
1017      droid_window_enqueue_buffer(disp, dri2_surf);
1018
1019   dri2_dpy->flush->invalidate(dri2_surf->dri_drawable);
1020
1021   /* Update the shared buffer mode */
1022   if (has_mutable_rb &&
1023       draw->ActiveRenderBuffer != draw->RequestedRenderBuffer) {
1024       bool mode = (draw->RequestedRenderBuffer == EGL_SINGLE_BUFFER);
1025      _eglLog(_EGL_DEBUG, "%s: change to shared buffer mode %d",
1026              __func__, mode);
1027
1028      if (!droid_set_shared_buffer_mode(disp, draw, mode))
1029         return EGL_FALSE;
1030      draw->ActiveRenderBuffer = draw->RequestedRenderBuffer;
1031   }
1032
1033   return EGL_TRUE;
1034}
1035
1036#ifdef HAVE_DRM_GRALLOC
1037static int get_format(int format)
1038{
1039   switch (format) {
1040   case HAL_PIXEL_FORMAT_BGRA_8888: return __DRI_IMAGE_FORMAT_ARGB8888;
1041   case HAL_PIXEL_FORMAT_RGB_565:   return __DRI_IMAGE_FORMAT_RGB565;
1042   case HAL_PIXEL_FORMAT_RGBA_8888: return __DRI_IMAGE_FORMAT_ABGR8888;
1043   case HAL_PIXEL_FORMAT_IMPLEMENTATION_DEFINED:
1044      /*
1045       * HACK: Hardcode this to RGBX_8888 as per cros_gralloc hack.
1046       * TODO: Revert this once https://issuetracker.google.com/32077885 is fixed.
1047       */
1048   case HAL_PIXEL_FORMAT_RGBX_8888: return __DRI_IMAGE_FORMAT_XBGR8888;
1049   case HAL_PIXEL_FORMAT_RGBA_FP16: return __DRI_IMAGE_FORMAT_ABGR16161616F;
1050   case HAL_PIXEL_FORMAT_RGBA_1010102: return __DRI_IMAGE_FORMAT_ABGR2101010;
1051   default:
1052      _eglLog(_EGL_WARNING, "unsupported native buffer format 0x%x", format);
1053   }
1054   return -1;
1055}
1056
1057static __DRIimage *
1058droid_create_image_from_name(_EGLDisplay *disp,
1059                             struct ANativeWindowBuffer *buf,
1060                             void *priv)
1061{
1062   struct dri2_egl_display *dri2_dpy = dri2_egl_display(disp);
1063   int name;
1064   int format;
1065
1066   name = get_native_buffer_name(buf);
1067   if (!name) {
1068      _eglError(EGL_BAD_PARAMETER, "eglCreateEGLImageKHR");
1069      return NULL;
1070   }
1071
1072   format = get_format(buf->format);
1073   if (format == -1)
1074       return NULL;
1075
1076   return
1077      dri2_dpy->image->createImageFromName(dri2_dpy->dri_screen,
1078					   buf->width,
1079					   buf->height,
1080					   format,
1081					   name,
1082					   buf->stride,
1083					   priv);
1084}
1085#endif /* HAVE_DRM_GRALLOC */
1086
1087static EGLBoolean
1088droid_query_surface(_EGLDisplay *disp, _EGLSurface *surf,
1089                    EGLint attribute, EGLint *value)
1090{
1091   struct dri2_egl_surface *dri2_surf = dri2_egl_surface(surf);
1092   switch (attribute) {
1093      case EGL_WIDTH:
1094         if (dri2_surf->base.Type == EGL_WINDOW_BIT && dri2_surf->window) {
1095            ANativeWindow_query(dri2_surf->window,
1096                                ANATIVEWINDOW_QUERY_DEFAULT_WIDTH, value);
1097            return EGL_TRUE;
1098         }
1099         break;
1100      case EGL_HEIGHT:
1101         if (dri2_surf->base.Type == EGL_WINDOW_BIT && dri2_surf->window) {
1102            ANativeWindow_query(dri2_surf->window,
1103                                ANATIVEWINDOW_QUERY_DEFAULT_HEIGHT, value);
1104            return EGL_TRUE;
1105         }
1106         break;
1107      default:
1108         break;
1109   }
1110   return _eglQuerySurface(disp, surf, attribute, value);
1111}
1112
1113static _EGLImage *
1114dri2_create_image_android_native_buffer(_EGLDisplay *disp,
1115                                        _EGLContext *ctx,
1116                                        struct ANativeWindowBuffer *buf)
1117{
1118   if (ctx != NULL) {
1119      /* From the EGL_ANDROID_image_native_buffer spec:
1120       *
1121       *     * If <target> is EGL_NATIVE_BUFFER_ANDROID and <ctx> is not
1122       *       EGL_NO_CONTEXT, the error EGL_BAD_CONTEXT is generated.
1123       */
1124      _eglError(EGL_BAD_CONTEXT, "eglCreateEGLImageKHR: for "
1125                "EGL_NATIVE_BUFFER_ANDROID, the context must be "
1126                "EGL_NO_CONTEXT");
1127      return NULL;
1128   }
1129
1130   if (!buf || buf->common.magic != ANDROID_NATIVE_BUFFER_MAGIC ||
1131       buf->common.version != sizeof(*buf)) {
1132      _eglError(EGL_BAD_PARAMETER, "eglCreateEGLImageKHR");
1133      return NULL;
1134   }
1135
1136   __DRIimage *dri_image =
1137      droid_create_image_from_native_buffer(disp, buf, buf);
1138
1139#ifdef HAVE_DRM_GRALLOC
1140   if (dri_image == NULL)
1141      dri_image = droid_create_image_from_name(disp, buf, buf);
1142#endif
1143
1144   if (dri_image) {
1145#if ANDROID_API_LEVEL >= 26
1146      AHardwareBuffer_acquire(ANativeWindowBuffer_getHardwareBuffer(buf));
1147#endif
1148      return dri2_create_image_from_dri(disp, dri_image);
1149   }
1150
1151   return NULL;
1152}
1153
1154static _EGLImage *
1155droid_create_image_khr(_EGLDisplay *disp, _EGLContext *ctx, EGLenum target,
1156                       EGLClientBuffer buffer, const EGLint *attr_list)
1157{
1158   switch (target) {
1159   case EGL_NATIVE_BUFFER_ANDROID:
1160      return dri2_create_image_android_native_buffer(disp, ctx,
1161            (struct ANativeWindowBuffer *) buffer);
1162   default:
1163      return dri2_create_image_khr(disp, ctx, target, buffer, attr_list);
1164   }
1165}
1166
1167static void
1168droid_flush_front_buffer(__DRIdrawable * driDrawable, void *loaderPrivate)
1169{
1170}
1171
1172#ifdef HAVE_DRM_GRALLOC
1173static int
1174droid_get_buffers_parse_attachments(struct dri2_egl_surface *dri2_surf,
1175                                    unsigned int *attachments, int count)
1176{
1177   int num_buffers = 0;
1178
1179   /* fill dri2_surf->buffers */
1180   for (int i = 0; i < count * 2; i += 2) {
1181      __DRIbuffer *buf, *local;
1182
1183      assert(num_buffers < ARRAY_SIZE(dri2_surf->buffers));
1184      buf = &dri2_surf->buffers[num_buffers];
1185
1186      switch (attachments[i]) {
1187      case __DRI_BUFFER_BACK_LEFT:
1188         if (dri2_surf->base.Type == EGL_WINDOW_BIT) {
1189            buf->attachment = attachments[i];
1190            buf->name = get_native_buffer_name(dri2_surf->buffer);
1191            buf->cpp = get_format_bpp(dri2_surf->buffer->format);
1192            buf->pitch = dri2_surf->buffer->stride * buf->cpp;
1193            buf->flags = 0;
1194
1195            if (buf->name)
1196               num_buffers++;
1197
1198            break;
1199         }
1200         FALLTHROUGH; /* for pbuffers */
1201      case __DRI_BUFFER_DEPTH:
1202      case __DRI_BUFFER_STENCIL:
1203      case __DRI_BUFFER_ACCUM:
1204      case __DRI_BUFFER_DEPTH_STENCIL:
1205      case __DRI_BUFFER_HIZ:
1206         local = dri2_egl_surface_alloc_local_buffer(dri2_surf,
1207               attachments[i], attachments[i + 1]);
1208
1209         if (local) {
1210            *buf = *local;
1211            num_buffers++;
1212         }
1213         break;
1214      case __DRI_BUFFER_FRONT_LEFT:
1215      case __DRI_BUFFER_FRONT_RIGHT:
1216      case __DRI_BUFFER_FAKE_FRONT_LEFT:
1217      case __DRI_BUFFER_FAKE_FRONT_RIGHT:
1218      case __DRI_BUFFER_BACK_RIGHT:
1219      default:
1220         /* no front or right buffers */
1221         break;
1222      }
1223   }
1224
1225   return num_buffers;
1226}
1227
1228static __DRIbuffer *
1229droid_get_buffers_with_format(__DRIdrawable * driDrawable,
1230			     int *width, int *height,
1231			     unsigned int *attachments, int count,
1232			     int *out_count, void *loaderPrivate)
1233{
1234   struct dri2_egl_surface *dri2_surf = loaderPrivate;
1235
1236   if (update_buffers(dri2_surf) < 0)
1237      return NULL;
1238
1239   *out_count = droid_get_buffers_parse_attachments(dri2_surf, attachments, count);
1240
1241   if (width)
1242      *width = dri2_surf->base.Width;
1243   if (height)
1244      *height = dri2_surf->base.Height;
1245
1246   return dri2_surf->buffers;
1247}
1248#endif /* HAVE_DRM_GRALLOC */
1249
1250static unsigned
1251droid_get_capability(void *loaderPrivate, enum dri_loader_cap cap)
1252{
1253   /* Note: loaderPrivate is _EGLDisplay* */
1254   switch (cap) {
1255   case DRI_LOADER_CAP_RGBA_ORDERING:
1256      return 1;
1257   default:
1258      return 0;
1259   }
1260}
1261
1262static void
1263droid_destroy_loader_image_state(void *loaderPrivate)
1264{
1265#if ANDROID_API_LEVEL >= 26
1266   if (loaderPrivate) {
1267      AHardwareBuffer_release(
1268            ANativeWindowBuffer_getHardwareBuffer(loaderPrivate));
1269   }
1270#endif
1271}
1272
1273static EGLBoolean
1274droid_add_configs_for_visuals(_EGLDisplay *disp)
1275{
1276   struct dri2_egl_display *dri2_dpy = dri2_egl_display(disp);
1277   static const struct {
1278      int format;
1279      int rgba_shifts[4];
1280      unsigned int rgba_sizes[4];
1281   } visuals[] = {
1282      { HAL_PIXEL_FORMAT_RGBA_8888, { 0, 8, 16, 24 }, { 8, 8, 8, 8 } },
1283      { HAL_PIXEL_FORMAT_RGBX_8888, { 0, 8, 16, -1 }, { 8, 8, 8, 0 } },
1284      { HAL_PIXEL_FORMAT_RGB_565,   { 11, 5, 0, -1 }, { 5, 6, 5, 0 } },
1285      /* This must be after HAL_PIXEL_FORMAT_RGBA_8888, we only keep BGRA
1286       * visual if it turns out RGBA visual is not available.
1287       */
1288      { HAL_PIXEL_FORMAT_BGRA_8888, { 16, 8, 0, 24 }, { 8, 8, 8, 8 } },
1289   };
1290
1291   unsigned int format_count[ARRAY_SIZE(visuals)] = { 0 };
1292   int config_count = 0;
1293
1294   /* The nesting of loops is significant here. Also significant is the order
1295    * of the HAL pixel formats. Many Android apps (such as Google's official
1296    * NDK GLES2 example app), and even portions the core framework code (such
1297    * as SystemServiceManager in Nougat), incorrectly choose their EGLConfig.
1298    * They neglect to match the EGLConfig's EGL_NATIVE_VISUAL_ID against the
1299    * window's native format, and instead choose the first EGLConfig whose
1300    * channel sizes match those of the native window format while ignoring the
1301    * channel *ordering*.
1302    *
1303    * We can detect such buggy clients in logcat when they call
1304    * eglCreateSurface, by detecting the mismatch between the EGLConfig's
1305    * format and the window's format.
1306    *
1307    * As a workaround, we generate EGLConfigs such that all EGLConfigs for HAL
1308    * pixel format i precede those for HAL pixel format i+1. In my
1309    * (chadversary) testing on Android Nougat, this was good enough to pacify
1310    * the buggy clients.
1311    */
1312   bool has_rgba = false;
1313   for (int i = 0; i < ARRAY_SIZE(visuals); i++) {
1314      /* Only enable BGRA configs when RGBA is not available. BGRA configs are
1315       * buggy on stock Android.
1316       */
1317      if (visuals[i].format == HAL_PIXEL_FORMAT_BGRA_8888 && has_rgba)
1318         continue;
1319      for (int j = 0; dri2_dpy->driver_configs[j]; j++) {
1320         const EGLint surface_type = EGL_WINDOW_BIT | EGL_PBUFFER_BIT;
1321
1322         const EGLint config_attrs[] = {
1323           EGL_NATIVE_VISUAL_ID,   visuals[i].format,
1324           EGL_NATIVE_VISUAL_TYPE, visuals[i].format,
1325           EGL_FRAMEBUFFER_TARGET_ANDROID, EGL_TRUE,
1326           EGL_RECORDABLE_ANDROID, EGL_TRUE,
1327           EGL_NONE
1328         };
1329
1330         struct dri2_egl_config *dri2_conf =
1331            dri2_add_config(disp, dri2_dpy->driver_configs[j],
1332                            config_count + 1, surface_type, config_attrs,
1333                            visuals[i].rgba_shifts, visuals[i].rgba_sizes);
1334         if (dri2_conf) {
1335            if (dri2_conf->base.ConfigID == config_count + 1)
1336               config_count++;
1337            format_count[i]++;
1338         }
1339      }
1340      if (visuals[i].format == HAL_PIXEL_FORMAT_RGBA_8888 && format_count[i])
1341         has_rgba = true;
1342   }
1343
1344   for (int i = 0; i < ARRAY_SIZE(format_count); i++) {
1345      if (!format_count[i]) {
1346         _eglLog(_EGL_DEBUG, "No DRI config supports native format 0x%x",
1347                 visuals[i].format);
1348      }
1349   }
1350
1351   return (config_count != 0);
1352}
1353
1354static const struct dri2_egl_display_vtbl droid_display_vtbl = {
1355   .authenticate = NULL,
1356   .create_window_surface = droid_create_window_surface,
1357   .create_pbuffer_surface = droid_create_pbuffer_surface,
1358   .destroy_surface = droid_destroy_surface,
1359   .create_image = droid_create_image_khr,
1360   .swap_buffers = droid_swap_buffers,
1361   .swap_interval = droid_swap_interval,
1362   .query_buffer_age = droid_query_buffer_age,
1363   .query_surface = droid_query_surface,
1364   .get_dri_drawable = dri2_surface_get_dri_drawable,
1365   .set_shared_buffer_mode = droid_set_shared_buffer_mode,
1366};
1367
1368#ifdef HAVE_DRM_GRALLOC
1369static const __DRIdri2LoaderExtension droid_dri2_loader_extension = {
1370   .base = { __DRI_DRI2_LOADER, 5 },
1371
1372   .getBuffers               = NULL,
1373   .flushFrontBuffer         = droid_flush_front_buffer,
1374   .getBuffersWithFormat     = droid_get_buffers_with_format,
1375   .getCapability            = droid_get_capability,
1376   .destroyLoaderImageState  = droid_destroy_loader_image_state,
1377};
1378
1379static const __DRIextension *droid_dri2_loader_extensions[] = {
1380   &droid_dri2_loader_extension.base,
1381   &image_lookup_extension.base,
1382   &use_invalidate.base,
1383   /* No __DRI_MUTABLE_RENDER_BUFFER_LOADER because it requires
1384    * __DRI_IMAGE_LOADER.
1385    */
1386   NULL,
1387};
1388#endif /* HAVE_DRM_GRALLOC */
1389
1390static const __DRIimageLoaderExtension droid_image_loader_extension = {
1391   .base = { __DRI_IMAGE_LOADER, 4 },
1392
1393   .getBuffers               = droid_image_get_buffers,
1394   .flushFrontBuffer         = droid_flush_front_buffer,
1395   .getCapability            = droid_get_capability,
1396   .flushSwapBuffers         = NULL,
1397   .destroyLoaderImageState  = droid_destroy_loader_image_state,
1398};
1399
1400static void
1401droid_display_shared_buffer(__DRIdrawable *driDrawable, int fence_fd,
1402                            void *loaderPrivate)
1403{
1404   struct dri2_egl_surface *dri2_surf = loaderPrivate;
1405   struct ANativeWindowBuffer *old_buffer UNUSED = dri2_surf->buffer;
1406
1407   if (!_eglSurfaceInSharedBufferMode(&dri2_surf->base)) {
1408      _eglLog(_EGL_WARNING, "%s: internal error: buffer is not shared",
1409              __func__);
1410      return;
1411   }
1412
1413   if (fence_fd >= 0) {
1414      /* The driver's fence is more recent than the surface's out fence, if it
1415       * exists at all. So use the driver's fence.
1416       */
1417      if (dri2_surf->out_fence_fd >= 0) {
1418         close(dri2_surf->out_fence_fd);
1419         dri2_surf->out_fence_fd = -1;
1420      }
1421   } else if (dri2_surf->out_fence_fd >= 0) {
1422      fence_fd = dri2_surf->out_fence_fd;
1423      dri2_surf->out_fence_fd = -1;
1424   }
1425
1426   if (ANativeWindow_queueBuffer(dri2_surf->window, dri2_surf->buffer,
1427                                 fence_fd)) {
1428      _eglLog(_EGL_WARNING, "%s: ANativeWindow_queueBuffer failed", __func__);
1429      close(fence_fd);
1430      return;
1431   }
1432
1433   fence_fd = -1;
1434
1435   if (ANativeWindow_dequeueBuffer(dri2_surf->window, &dri2_surf->buffer,
1436                                   &fence_fd)) {
1437      /* Tear down the surface because it no longer has a back buffer. */
1438      struct dri2_egl_display *dri2_dpy =
1439         dri2_egl_display(dri2_surf->base.Resource.Display);
1440
1441      _eglLog(_EGL_WARNING, "%s: ANativeWindow_dequeueBuffer failed", __func__);
1442
1443      dri2_surf->base.Lost = true;
1444      dri2_surf->buffer = NULL;
1445      dri2_surf->back = NULL;
1446
1447      if (dri2_surf->dri_image_back) {
1448         dri2_dpy->image->destroyImage(dri2_surf->dri_image_back);
1449         dri2_surf->dri_image_back = NULL;
1450      }
1451
1452      dri2_dpy->flush->invalidate(dri2_surf->dri_drawable);
1453      return;
1454   }
1455
1456   close_in_fence_fd(dri2_surf);
1457   validate_fence_fd(fence_fd);
1458   dri2_surf->in_fence_fd = fence_fd;
1459   handle_in_fence_fd(dri2_surf, dri2_surf->dri_image_back);
1460}
1461
1462static const __DRImutableRenderBufferLoaderExtension droid_mutable_render_buffer_extension = {
1463   .base = { __DRI_MUTABLE_RENDER_BUFFER_LOADER, 1 },
1464   .displaySharedBuffer = droid_display_shared_buffer,
1465};
1466
1467static const __DRIextension *droid_image_loader_extensions[] = {
1468   &droid_image_loader_extension.base,
1469   &image_lookup_extension.base,
1470   &use_invalidate.base,
1471   &droid_mutable_render_buffer_extension.base,
1472   NULL,
1473};
1474
1475static EGLBoolean
1476droid_load_driver(_EGLDisplay *disp, bool swrast)
1477{
1478   struct dri2_egl_display *dri2_dpy = dri2_egl_display(disp);
1479
1480   dri2_dpy->driver_name = loader_get_driver_for_fd(dri2_dpy->fd);
1481   if (dri2_dpy->driver_name == NULL)
1482      return false;
1483
1484#ifdef HAVE_DRM_GRALLOC
1485   /* Handle control nodes using __DRI_DRI2_LOADER extension and GEM names
1486    * for backwards compatibility with drm_gralloc. (Do not use on new
1487    * systems.) */
1488   dri2_dpy->loader_extensions = droid_dri2_loader_extensions;
1489   if (!dri2_load_driver(disp)) {
1490      goto error;
1491   }
1492#else
1493   if (swrast) {
1494      /* Use kms swrast only with vgem / virtio_gpu.
1495       * virtio-gpu fallbacks to software rendering when 3D features
1496       * are unavailable since 6c5ab.
1497       */
1498      if (strcmp(dri2_dpy->driver_name, "vgem") == 0 ||
1499          strcmp(dri2_dpy->driver_name, "virtio_gpu") == 0) {
1500         free(dri2_dpy->driver_name);
1501         dri2_dpy->driver_name = strdup("kms_swrast");
1502      } else {
1503         goto error;
1504      }
1505   }
1506
1507   dri2_dpy->loader_extensions = droid_image_loader_extensions;
1508   if (!dri2_load_driver_dri3(disp)) {
1509      goto error;
1510   }
1511#endif
1512
1513   return true;
1514
1515error:
1516   free(dri2_dpy->driver_name);
1517   dri2_dpy->driver_name = NULL;
1518   return false;
1519}
1520
1521static void
1522droid_unload_driver(_EGLDisplay *disp)
1523{
1524   struct dri2_egl_display *dri2_dpy = dri2_egl_display(disp);
1525
1526   dlclose(dri2_dpy->driver);
1527   dri2_dpy->driver = NULL;
1528   free(dri2_dpy->driver_name);
1529   dri2_dpy->driver_name = NULL;
1530}
1531
1532static int
1533droid_filter_device(_EGLDisplay *disp, int fd, const char *vendor)
1534{
1535   drmVersionPtr ver = drmGetVersion(fd);
1536   if (!ver)
1537      return -1;
1538
1539   if (strcmp(vendor, ver->name) != 0) {
1540      drmFreeVersion(ver);
1541      return -1;
1542   }
1543
1544   drmFreeVersion(ver);
1545   return 0;
1546}
1547
1548static EGLBoolean
1549droid_probe_device(_EGLDisplay *disp, bool swrast)
1550{
1551  /* Check that the device is supported, by attempting to:
1552   * - load the dri module
1553   * - and, create a screen
1554   */
1555   if (!droid_load_driver(disp, swrast))
1556      return EGL_FALSE;
1557
1558   if (!dri2_create_screen(disp)) {
1559      _eglLog(_EGL_WARNING, "DRI2: failed to create screen");
1560      droid_unload_driver(disp);
1561      return EGL_FALSE;
1562   }
1563   return EGL_TRUE;
1564}
1565
1566#ifdef HAVE_DRM_GRALLOC
1567static EGLBoolean
1568droid_open_device(_EGLDisplay *disp, bool swrast)
1569{
1570   struct dri2_egl_display *dri2_dpy = dri2_egl_display(disp);
1571   int fd = -1, err = -EINVAL;
1572
1573   if (swrast)
1574      return EGL_FALSE;
1575
1576   if (dri2_dpy->gralloc->perform)
1577      err = dri2_dpy->gralloc->perform(dri2_dpy->gralloc,
1578                                       GRALLOC_MODULE_PERFORM_GET_DRM_FD,
1579                                       &fd);
1580   if (err || fd < 0) {
1581      _eglLog(_EGL_WARNING, "fail to get drm fd");
1582      return EGL_FALSE;
1583   }
1584
1585   dri2_dpy->fd = os_dupfd_cloexec(fd);
1586   if (dri2_dpy->fd < 0)
1587      return EGL_FALSE;
1588
1589   if (drmGetNodeTypeFromFd(dri2_dpy->fd) == DRM_NODE_RENDER)
1590      return EGL_FALSE;
1591
1592   return droid_probe_device(disp, swrast);
1593}
1594#else
1595static EGLBoolean
1596droid_open_device(_EGLDisplay *disp, bool swrast)
1597{
1598#define MAX_DRM_DEVICES 64
1599   struct dri2_egl_display *dri2_dpy = dri2_egl_display(disp);
1600   drmDevicePtr device, devices[MAX_DRM_DEVICES] = { NULL };
1601   int num_devices;
1602
1603   char *vendor_name = NULL;
1604   char vendor_buf[PROPERTY_VALUE_MAX];
1605
1606#ifdef EGL_FORCE_RENDERNODE
1607   const unsigned node_type = DRM_NODE_RENDER;
1608#else
1609   const unsigned node_type = swrast ? DRM_NODE_PRIMARY : DRM_NODE_RENDER;
1610#endif
1611
1612   if (property_get("drm.gpu.vendor_name", vendor_buf, NULL) > 0)
1613      vendor_name = vendor_buf;
1614
1615   num_devices = drmGetDevices2(0, devices, ARRAY_SIZE(devices));
1616   if (num_devices < 0)
1617      return EGL_FALSE;
1618
1619   for (int i = 0; i < num_devices; i++) {
1620      device = devices[i];
1621
1622      if (!(device->available_nodes & (1 << node_type)))
1623         continue;
1624
1625      dri2_dpy->fd = loader_open_device(device->nodes[node_type]);
1626      if (dri2_dpy->fd < 0) {
1627         _eglLog(_EGL_WARNING, "%s() Failed to open DRM device %s",
1628                 __func__, device->nodes[node_type]);
1629         continue;
1630      }
1631
1632      /* If a vendor is explicitly provided, we use only that.
1633       * Otherwise we fall-back the first device that is supported.
1634       */
1635      if (vendor_name) {
1636         if (droid_filter_device(disp, dri2_dpy->fd, vendor_name)) {
1637            /* Device does not match - try next device */
1638            close(dri2_dpy->fd);
1639            dri2_dpy->fd = -1;
1640            continue;
1641         }
1642         /* If the requested device matches - use it. Regardless if
1643          * init fails, do not fall-back to any other device.
1644          */
1645         if (!droid_probe_device(disp, false)) {
1646            close(dri2_dpy->fd);
1647            dri2_dpy->fd = -1;
1648         }
1649
1650         break;
1651      }
1652      if (droid_probe_device(disp, swrast))
1653         break;
1654
1655      /* No explicit request - attempt the next device */
1656      close(dri2_dpy->fd);
1657      dri2_dpy->fd = -1;
1658   }
1659   drmFreeDevices(devices, num_devices);
1660
1661   if (dri2_dpy->fd < 0) {
1662      _eglLog(_EGL_WARNING, "Failed to open %s DRM device",
1663            vendor_name ? "desired": "any");
1664      return EGL_FALSE;
1665   }
1666
1667   return EGL_TRUE;
1668#undef MAX_DRM_DEVICES
1669}
1670
1671#endif
1672
1673EGLBoolean
1674dri2_initialize_android(_EGLDisplay *disp)
1675{
1676   _EGLDevice *dev;
1677   bool device_opened = false;
1678   struct dri2_egl_display *dri2_dpy;
1679   const char *err;
1680   int ret;
1681
1682   dri2_dpy = calloc(1, sizeof(*dri2_dpy));
1683   if (!dri2_dpy)
1684      return _eglError(EGL_BAD_ALLOC, "eglInitialize");
1685
1686   dri2_dpy->fd = -1;
1687   ret = hw_get_module(GRALLOC_HARDWARE_MODULE_ID,
1688                       (const hw_module_t **)&dri2_dpy->gralloc);
1689   if (ret) {
1690      err = "DRI2: failed to get gralloc module";
1691      goto cleanup;
1692   }
1693
1694   disp->DriverData = (void *) dri2_dpy;
1695   device_opened = droid_open_device(disp, disp->Options.ForceSoftware);
1696
1697   if (!device_opened) {
1698      err = "DRI2: failed to open device";
1699      goto cleanup;
1700   }
1701
1702   dev = _eglAddDevice(dri2_dpy->fd, false);
1703   if (!dev) {
1704      err = "DRI2: failed to find EGLDevice";
1705      goto cleanup;
1706   }
1707
1708   disp->Device = dev;
1709
1710   if (!dri2_setup_extensions(disp)) {
1711      err = "DRI2: failed to setup extensions";
1712      goto cleanup;
1713   }
1714
1715   dri2_setup_screen(disp);
1716
1717   /* We set the maximum swap interval as 1 for Android platform, since it is
1718    * the maximum value supported by Android according to the value of
1719    * ANativeWindow::maxSwapInterval.
1720    */
1721   dri2_setup_swap_interval(disp, 1);
1722
1723   disp->Extensions.ANDROID_framebuffer_target = EGL_TRUE;
1724   disp->Extensions.ANDROID_image_native_buffer = EGL_TRUE;
1725   disp->Extensions.ANDROID_recordable = EGL_TRUE;
1726
1727   /* Querying buffer age requires a buffer to be dequeued.  Without
1728    * EGL_ANDROID_native_fence_sync, dequeue might call eglClientWaitSync and
1729    * result in a deadlock (the lock is already held by eglQuerySurface).
1730    */
1731   if (disp->Extensions.ANDROID_native_fence_sync) {
1732      disp->Extensions.EXT_buffer_age = EGL_TRUE;
1733   } else {
1734      /* disable KHR_partial_update that might have been enabled in
1735       * dri2_setup_screen
1736       */
1737      disp->Extensions.KHR_partial_update = EGL_FALSE;
1738   }
1739
1740   disp->Extensions.KHR_image = EGL_TRUE;
1741
1742   dri2_dpy->front_rendering_usage = 0;
1743#if ANDROID_API_LEVEL >= 24
1744   if (dri2_dpy->mutable_render_buffer &&
1745       dri2_dpy->loader_extensions == droid_image_loader_extensions &&
1746       /* In big GL, front rendering is done at the core API level by directly
1747        * rendering on the front buffer. However, in ES, the front buffer is
1748        * completely inaccessible through the core ES API.
1749        *
1750        * EGL_KHR_mutable_render_buffer is Android's attempt to re-introduce
1751        * front rendering into ES by squeezing into EGL. Unlike big GL, this
1752        * extension redirects GL_BACK used by ES for front rendering. Thus we
1753        * restrict the enabling of this extension to ES only.
1754        */
1755       (disp->ClientAPIs & ~(EGL_OPENGL_ES_BIT | EGL_OPENGL_ES2_BIT |
1756                             EGL_OPENGL_ES3_BIT_KHR)) == 0) {
1757      /* For cros gralloc, if the front rendering query is supported, then all
1758       * available window surface configs support front rendering because:
1759       *
1760       * 1) EGL queries cros gralloc for the front rendering usage bit here
1761       * 2) EGL combines the front rendering usage bit with the existing usage
1762       *    if the window surface requests mutable render buffer
1763       * 3) EGL sets the combined usage onto the ANativeWindow and the next
1764       *    dequeueBuffer will ask gralloc for an allocation/re-allocation with
1765       *    the new combined usage
1766       * 4) cros gralloc(on top of minigbm) resolves the front rendering usage
1767       *    bit into either BO_USE_FRONT_RENDERING or BO_USE_LINEAR based on
1768       *    the format support checking.
1769       *
1770       * So at least we can force BO_USE_LINEAR as the fallback.
1771       */
1772      uint32_t front_rendering_usage = 0;
1773      if (!strcmp(dri2_dpy->gralloc->common.name, cros_gralloc_module_name) &&
1774          dri2_dpy->gralloc->perform &&
1775          dri2_dpy->gralloc->perform(
1776                dri2_dpy->gralloc, CROS_GRALLOC_DRM_GET_USAGE,
1777                CROS_GRALLOC_DRM_GET_USAGE_FRONT_RENDERING_BIT,
1778                &front_rendering_usage) == 0) {
1779         dri2_dpy->front_rendering_usage = front_rendering_usage;
1780         disp->Extensions.KHR_mutable_render_buffer = EGL_TRUE;
1781      }
1782   }
1783#endif
1784
1785   /* Create configs *after* enabling extensions because presence of DRI
1786    * driver extensions can affect the capabilities of EGLConfigs.
1787    */
1788   if (!droid_add_configs_for_visuals(disp)) {
1789      err = "DRI2: failed to add configs";
1790      goto cleanup;
1791   }
1792
1793   /* Fill vtbl last to prevent accidentally calling virtual function during
1794    * initialization.
1795    */
1796   dri2_dpy->vtbl = &droid_display_vtbl;
1797
1798   return EGL_TRUE;
1799
1800cleanup:
1801   dri2_display_destroy(disp);
1802   return _eglError(EGL_NOT_INITIALIZED, err);
1803}
1804