Lines Matching defs:pcl
111 static inline bool z_erofs_is_inline_pcluster(struct z_erofs_pcluster *pcl)
113 return !pcl->obj.index;
116 static inline unsigned int z_erofs_pclusterpages(struct z_erofs_pcluster *pcl)
118 if (z_erofs_is_inline_pcluster(pcl))
120 return pcl->pclusterpages;
307 struct z_erofs_pcluster *pcl;
312 pcl = kmem_cache_zalloc(pcs->slab, GFP_NOFS);
313 if (!pcl)
315 pcl->pclusterpages = nrpages;
316 return pcl;
321 static void z_erofs_free_pcluster(struct z_erofs_pcluster *pcl)
323 unsigned int pclusterpages = z_erofs_pclusterpages(pcl);
332 kmem_cache_free(pcs->slab, pcl);
512 * | (of the current pcl) | (of the previous pcl) |
527 struct z_erofs_pcluster *pcl;
561 struct z_erofs_pcluster *pcl = fe->pcl;
575 for (i = 0; i < pcl->pclusterpages; ++i) {
581 if (READ_ONCE(pcl->compressed_bvecs[i].page))
584 page = find_get_page(mc, pcl->obj.index + i);
606 if (!cmpxchg_relaxed(&pcl->compressed_bvecs[i].page, NULL, t))
627 struct z_erofs_pcluster *const pcl =
631 DBG_BUGON(z_erofs_is_inline_pcluster(pcl));
636 for (i = 0; i < pcl->pclusterpages; ++i) {
637 struct page *page = pcl->compressed_bvecs[i].page;
650 WRITE_ONCE(pcl->compressed_bvecs[i].page, NULL);
659 struct z_erofs_pcluster *pcl = folio_get_private(folio);
667 spin_lock(&pcl->obj.lockref.lock);
668 if (pcl->obj.lockref.count > 0)
671 DBG_BUGON(z_erofs_is_inline_pcluster(pcl));
672 for (i = 0; i < pcl->pclusterpages; ++i) {
673 if (pcl->compressed_bvecs[i].page == &folio->page) {
674 WRITE_ONCE(pcl->compressed_bvecs[i].page, NULL);
682 spin_unlock(&pcl->obj.lockref.lock);
727 struct z_erofs_pcluster *const pcl = fe->pcl;
730 if (!cmpxchg(&pcl->compressed_bvecs[--fe->icur].page,
732 pcl->compressed_bvecs[fe->icur] = *bvec;
756 fe->pcl->vcnt += (ret >= 0);
762 struct z_erofs_pcluster *pcl = f->pcl;
766 if (cmpxchg(&pcl->next, Z_EROFS_PCLUSTER_NIL,
768 *owned_head = &pcl->next;
782 struct z_erofs_pcluster *pcl;
793 pcl = z_erofs_alloc_pcluster(ztailpacking ? 1 :
795 if (IS_ERR(pcl))
796 return PTR_ERR(pcl);
798 spin_lock_init(&pcl->obj.lockref.lock);
799 pcl->obj.lockref.count = 1; /* one ref for this request */
800 pcl->algorithmformat = map->m_algorithmformat;
801 pcl->length = 0;
802 pcl->partial = true;
805 pcl->next = fe->owned_head;
806 pcl->pageofs_out = map->m_la & ~PAGE_MASK;
813 mutex_init(&pcl->lock);
814 DBG_BUGON(!mutex_trylock(&pcl->lock));
817 pcl->obj.index = 0; /* which indicates ztailpacking */
818 pcl->tailpacking_size = map->m_plen;
820 pcl->obj.index = map->m_pa >> PAGE_SHIFT;
822 grp = erofs_insert_workgroup(fe->inode->i_sb, &pcl->obj);
828 if (grp != &pcl->obj) {
829 fe->pcl = container_of(grp,
835 fe->owned_head = &pcl->next;
836 fe->pcl = pcl;
840 mutex_unlock(&pcl->lock);
841 z_erofs_free_pcluster(pcl);
853 DBG_BUGON(fe->pcl);
866 fe->pcl = container_of(grp, struct z_erofs_pcluster, obj);
873 mutex_lock(&fe->pcl->lock);
879 z_erofs_bvec_iter_begin(&fe->biter, &fe->pcl->bvset,
880 Z_EROFS_INLINE_BVECS, fe->pcl->vcnt);
881 if (!z_erofs_is_inline_pcluster(fe->pcl)) {
894 WRITE_ONCE(fe->pcl->compressed_bvecs[0].page, map->buf.page);
895 fe->pcl->pageofs_in = map->m_pa & ~PAGE_MASK;
899 fe->icur = z_erofs_pclusterpages(fe->pcl);
915 struct z_erofs_pcluster *const pcl =
918 call_rcu(&pcl->rcu, z_erofs_rcu_callback);
923 struct z_erofs_pcluster *pcl = fe->pcl;
925 if (!pcl)
929 mutex_unlock(&pcl->lock);
939 erofs_workgroup_put(&pcl->obj);
941 fe->pcl = NULL;
1017 if (!fe->pcl) {
1043 if (fe->pcl->pageofs_out != (map->m_la & ~PAGE_MASK))
1044 fe->pcl->multibases = true;
1045 if (fe->pcl->length < offset + end - map->m_la) {
1046 fe->pcl->length = offset + end - map->m_la;
1047 fe->pcl->pageofs_out = map->m_la & ~PAGE_MASK;
1051 fe->pcl->length == map->m_llen)
1052 fe->pcl->partial = false;
1090 struct z_erofs_pcluster *pcl;
1113 if (!((bvec->offset + be->pcl->pageofs_out) & ~PAGE_MASK) &&
1115 bvec->offset + bvec->end == be->pcl->length)) {
1116 pgnr = (bvec->offset + be->pcl->pageofs_out) >> PAGE_SHIFT;
1133 unsigned int off0 = be->pcl->pageofs_out;
1143 end = min_t(unsigned int, be->pcl->length - bvi->bvec.offset,
1174 struct z_erofs_pcluster *pcl = be->pcl;
1179 z_erofs_bvec_iter_begin(&biter, &pcl->bvset, Z_EROFS_INLINE_BVECS, 0);
1180 for (i = 0; i < pcl->vcnt; ++i) {
1200 struct z_erofs_pcluster *pcl = be->pcl;
1201 unsigned int pclusterpages = z_erofs_pclusterpages(pcl);
1206 struct z_erofs_bvec *bvec = &pcl->compressed_bvecs[i];
1216 if (z_erofs_is_inline_pcluster(pcl)) {
1243 struct z_erofs_pcluster *pcl = be->pcl;
1244 unsigned int pclusterpages = z_erofs_pclusterpages(pcl);
1246 &erofs_decompressors[pcl->algorithmformat];
1252 mutex_lock(&pcl->lock);
1253 be->nr_pages = PAGE_ALIGN(pcl->length + pcl->pageofs_out) >> PAGE_SHIFT;
1285 if (z_erofs_is_inline_pcluster(pcl))
1286 inputsize = pcl->tailpacking_size;
1294 .pageofs_in = pcl->pageofs_in,
1295 .pageofs_out = pcl->pageofs_out,
1297 .outputsize = pcl->length,
1298 .alg = pcl->algorithmformat,
1300 .partial_decoding = pcl->partial,
1301 .fillgaps = pcl->multibases,
1306 if (z_erofs_is_inline_pcluster(pcl)) {
1307 page = pcl->compressed_bvecs[0].page;
1308 WRITE_ONCE(pcl->compressed_bvecs[0].page, NULL);
1318 WRITE_ONCE(pcl->compressed_bvecs[i].page, NULL);
1342 pcl->length = 0;
1343 pcl->partial = true;
1344 pcl->multibases = false;
1345 pcl->bvset.nextpage = NULL;
1346 pcl->vcnt = 0;
1349 WRITE_ONCE(pcl->next, Z_EROFS_PCLUSTER_NIL);
1350 mutex_unlock(&pcl->lock);
1368 be.pcl = container_of(owned, struct z_erofs_pcluster, next);
1369 owned = READ_ONCE(be.pcl->next);
1372 if (z_erofs_is_inline_pcluster(be.pcl))
1373 z_erofs_free_pcluster(be.pcl);
1375 erofs_workgroup_put(&be.pcl->obj);
1438 static struct page *pickup_page_for_submission(struct z_erofs_pcluster *pcl,
1443 const pgoff_t index = pcl->obj.index;
1452 page = READ_ONCE(pcl->compressed_bvecs[nr].page);
1466 WRITE_ONCE(pcl->compressed_bvecs[nr].page, page);
1492 WRITE_ONCE(pcl->compressed_bvecs[nr].page, page);
1503 set_page_private(page, (unsigned long)pcl);
1527 if (oldpage != cmpxchg(&pcl->compressed_bvecs[nr].page,
1539 attach_page_private(page, pcl);
1584 static void move_to_bypass_jobqueue(struct z_erofs_pcluster *pcl,
1591 WRITE_ONCE(pcl->next, Z_EROFS_PCLUSTER_TAIL);
1594 WRITE_ONCE(*bypass_qtail, &pcl->next);
1596 qtail[JQ_BYPASS] = &pcl->next;
1656 struct z_erofs_pcluster *pcl;
1662 pcl = container_of(owned_head, struct z_erofs_pcluster, next);
1663 owned_head = READ_ONCE(pcl->next);
1665 if (z_erofs_is_inline_pcluster(pcl)) {
1666 move_to_bypass_jobqueue(pcl, qtail, owned_head);
1672 .m_pa = erofs_pos(sb, pcl->obj.index),
1677 end = cur + pcl->pclusterpages;
1682 page = pickup_page_for_submission(pcl, i++,
1725 qtail[JQ_SUBMIT] = &pcl->next;
1727 move_to_bypass_jobqueue(pcl, qtail, owned_head);