1#!/usr/bin/env python3
2# SPDX-License-Identifier: Apache-2.0
3# -----------------------------------------------------------------------------
4# Copyright 2020-2021 Arm Limited
5#
6# Licensed under the Apache License, Version 2.0 (the "License"); you may not
7# use this file except in compliance with the License. You may obtain a copy
8# of the License at:
9#
10#     http://www.apache.org/licenses/LICENSE-2.0
11#
12# Unless required by applicable law or agreed to in writing, software
13# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
14# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
15# License for the specific language governing permissions and limitations
16# under the License.
17# -----------------------------------------------------------------------------
18"""
19The ``astc_test_result_report.py`` script consolidates all current sets of
20reference results into a single report giving PSNR diffs (absolute) and
21performance diffs (relative speedup, 1 = no change).
22"""
23
24import re
25import os
26import sys
27
28
29import testlib.resultset as trs
30from collections import defaultdict as ddict
31
32
33CONFIG_FILTER = [
34    re.compile(r"^.*1\.7.*$"),
35    re.compile(r"^.*sse.*$")
36]
37
38TESTSET_FILTER = [
39    re.compile(r"^Small$"),
40    re.compile(r"^Frymire$"),
41]
42
43QUALITY_FILTER = [
44]
45
46BLOCKSIZE_FILTER = [
47    re.compile(r"^12x12$")
48]
49
50
51def find_reference_results():
52    """
53    Scrape the Test/Images directory for result CSV files and return an
54    mapping of the result sets.
55
56    Returns:
57        Returns a three deep tree of dictionaries, with the final dict
58        pointing at a `ResultSet` object. The hierarchy is:
59
60            imageSet => quality => encoder => result
61    """
62    scriptDir = os.path.dirname(__file__)
63    imageDir = os.path.join(scriptDir, "Images")
64
65    # Pattern for extracting useful data from the CSV file name
66    filePat = re.compile(r"astc_reference-(.+)_(.+)_results\.csv")
67
68    # Build a three level dictionary we can write into
69    results = ddict(lambda: ddict(lambda: ddict()))
70
71    # Final all CSVs, load them and store them in the dict tree
72    for root, dirs, files in os.walk(imageDir):
73        for name in files:
74            match = filePat.match(name)
75            if match:
76
77                # Skip results set in the filter
78                skip = [1 for filt in CONFIG_FILTER if filt.match(name)]
79                if skip:
80                    continue
81
82                fullPath = os.path.join(root, name)
83
84                encoder = match.group(1)
85                quality = match.group(2)
86                imageSet = os.path.basename(root)
87
88                # Skip results set in the filter
89                skip = [1 for filt in TESTSET_FILTER if filt.match(imageSet)]
90                if skip:
91                    continue
92
93                # Skip results set in the filter
94                skip = [1 for filt in QUALITY_FILTER if filt.match(quality)]
95                if skip:
96                    continue
97
98                testRef = trs.ResultSet(imageSet)
99                testRef.load_from_file(fullPath)
100
101                patchedRef = trs.ResultSet(imageSet)
102
103                for result in testRef.records:
104                    skip = [1 for filt in BLOCKSIZE_FILTER if filt.match(result.blkSz)]
105                    if not skip:
106                        patchedRef.add_record(result)
107
108                results[imageSet][quality]["ref-%s" % encoder] = patchedRef
109
110    return results
111
112
113class DeltaRecord():
114    """
115    Record a single image result from N different encoders.
116
117    Attributes:
118        imageSet: The image set this cme from.
119        quality: The compressor quality used.
120        encoders: The names of the encoders used. The first encoder in this
121            list will be used as the reference result.
122        records: The raw records for the encoders. The order of records in this
123            list matches the order of the `encoders` list.
124    """
125
126    def __init__(self, imageSet, quality, encoders, records):
127        self.imageSet = imageSet
128        self.quality = quality
129
130        self.encoders = list(encoders)
131        self.records = list(records)
132
133        assert(len(self.encoders) == len(self.records))
134
135    def get_delta_header(self, tag):
136        """
137        Get the delta encoding header.
138
139        Args:
140            tag: The field name to include in the tag.
141
142        Return:
143            The array of strings, providing the header names.
144        """
145        result = []
146
147        for encoder in self.encoders[1:]:
148            result.append("%s %s" % (tag, encoder))
149
150        return result
151
152    def get_abs_delta(self, field):
153        """
154        Get an absolute delta result.
155
156        Args:
157            field: The Record attribute name to diff.
158
159        Return:
160            The array of float delta values.
161        """
162        result = []
163
164        root = self.records[0]
165        for record in self.records[1:]:
166            result.append(getattr(record, field) - getattr(root, field))
167
168        return result
169
170    def get_rel_delta(self, field):
171        """
172        Get an relative delta result (score / ref).
173
174        Args:
175            field: The Record attribute name to diff.
176
177        Return:
178            The array of float delta values.
179        """
180        result = []
181
182        root = self.records[0]
183        for record in self.records[1:]:
184            result.append(getattr(record, field) / getattr(root, field))
185
186        return result
187
188    def get_irel_delta(self, field):
189        """
190        Get an inverse relative delta result (ref / score).
191
192        Args:
193            field: The Record attribute name to diff.
194
195        Return:
196            The array of float delta values.
197        """
198        return [1.0 / x for x in self.get_rel_delta(field)]
199
200    def get_full_row_header_csv(self):
201        """
202        Get a CSV encoded delta record header.
203
204        Return:
205            The string for the row.
206        """
207        rows = [
208            "Image Set",
209            "Quality",
210            "Size",
211            "Name"
212        ]
213
214        rows.append("")
215        rows.extend(self.get_delta_header("PSNR"))
216
217        rows.append("")
218        rows.extend(self.get_delta_header("Speed"))
219
220        return ",".join(rows)
221
222    def get_full_row_csv(self):
223        """
224        Get a CSV encoded delta record.
225
226        Return:
227            The string for the row.
228        """
229        rows = [
230            self.imageSet,
231            self.quality,
232            self.records[0].name,
233            self.records[0].blkSz
234        ]
235
236        rows.append("")
237        data = ["%0.3f" % x for x in self.get_abs_delta("psnr")]
238        rows.extend(data)
239
240        rows.append("")
241        data = ["%0.3f" % x for x in self.get_irel_delta("cTime")]
242        rows.extend(data)
243
244        return ",".join(rows)
245
246
247def print_result_set(imageSet, quality, encoders, results, printHeader):
248    """
249    Attributes:
250        imageSet: The image set name.
251        quality: The compressor quality used.
252        encoders: The names of the encoders used. The first encoder in this
253            list will be used as the reference result.
254        results: The dict of results, indexed by encoder.
255        printHeader: True if the table header should be printed, else False.
256    """
257    results = [results[x] for x in encoders]
258    recordSizes = [len(x.records) for x in results]
259
260    # Skip result sets that are not the same size
261    # TODO: We can take the set intersection here to report what we can
262    if min(recordSizes) != max(recordSizes):
263        return
264
265    # Interleave all result records
266    recordSets = zip(*[x.records for x in results])
267
268    # Iterate each image
269    for recordSet in recordSets:
270        base = recordSet[0]
271
272        # Sanity check consistency
273        for record in recordSet[1:]:
274            assert(record.blkSz == base.blkSz)
275            assert(record.name == base.name)
276
277        dr = DeltaRecord(imageSet, quality, encoders, recordSet)
278
279        if printHeader:
280            print(dr.get_full_row_header_csv())
281            printHeader = False
282
283        print(dr.get_full_row_csv())
284
285
286def main():
287    """
288    The main function.
289
290    Returns:
291        int: The process return code.
292    """
293
294    results = find_reference_results()
295
296    imageSet = sorted(results.keys())
297
298    first = True
299    for image in imageSet:
300        qualityTree = results[image]
301        qualitySet = sorted(qualityTree.keys())
302
303        for qual in qualitySet:
304            encoderTree = qualityTree[qual]
305            encoderSet = sorted(encoderTree.keys())
306
307            if len(encoderSet) > 1:
308                print_result_set(image, qual, encoderSet, encoderTree, first)
309                first = False
310
311    return 0
312
313
314if __name__ == "__main__":
315    sys.exit(main())
316