1e5c31af7Sopenharmony_ci#!/usr/bin/python3 2e5c31af7Sopenharmony_ci# 3e5c31af7Sopenharmony_ci# Copyright 2016-2024 The Khronos Group Inc. 4e5c31af7Sopenharmony_ci# 5e5c31af7Sopenharmony_ci# SPDX-License-Identifier: Apache-2.0 6e5c31af7Sopenharmony_ci 7e5c31af7Sopenharmony_ci"""Used for automatic reflow of spec sources to satisfy the agreed layout to 8e5c31af7Sopenharmony_ciminimize git churn. Also used to insert identifying tags on explicit Valid 9e5c31af7Sopenharmony_ciUsage statements. 10e5c31af7Sopenharmony_ci 11e5c31af7Sopenharmony_ciUsage: `reflow.py [-noflow] [-tagvu] [-nextvu #] [-overwrite] [-out dir] [-suffix str] files` 12e5c31af7Sopenharmony_ci 13e5c31af7Sopenharmony_ci- `-noflow` acts as a passthrough, instead of reflowing text. Other 14e5c31af7Sopenharmony_ci processing may occur. 15e5c31af7Sopenharmony_ci- `-tagvu` generates explicit VUID tag for Valid Usage statements which 16e5c31af7Sopenharmony_ci do not already have them. 17e5c31af7Sopenharmony_ci- `-nextvu #` starts VUID tag generation at the specified # instead of 18e5c31af7Sopenharmony_ci the value wired into the `reflow.py` script. 19e5c31af7Sopenharmony_ci- `-overwrite` updates in place (can be risky, make sure there are backups) 20e5c31af7Sopenharmony_ci- `-check FAIL|WARN` runs some consistency checks on markup. If the checks 21e5c31af7Sopenharmony_ci fail and the WARN option is given, the script will simply print a warning 22e5c31af7Sopenharmony_ci message. If the checks fail and the FAIL option is given, the script will 23e5c31af7Sopenharmony_ci exit with an error code. FAIL is for use with continuous integration 24e5c31af7Sopenharmony_ci scripts enforcing the checks. 25e5c31af7Sopenharmony_ci- `-out` specifies directory to create output file in, default 'out' 26e5c31af7Sopenharmony_ci- `-suffix` specifies suffix to add to output files, default '' 27e5c31af7Sopenharmony_ci- `files` are asciidoc source files from the spec to reflow. 28e5c31af7Sopenharmony_ci""" 29e5c31af7Sopenharmony_ci# For error and file-loading interfaces only 30e5c31af7Sopenharmony_ciimport argparse 31e5c31af7Sopenharmony_ciimport os 32e5c31af7Sopenharmony_ciimport re 33e5c31af7Sopenharmony_ciimport sys 34e5c31af7Sopenharmony_cifrom reflib import loadFile, logDiag, logWarn, logErr, setLogFile, getBranch 35e5c31af7Sopenharmony_cifrom pathlib import Path 36e5c31af7Sopenharmony_ciimport doctransformer 37e5c31af7Sopenharmony_ci 38e5c31af7Sopenharmony_ci# Vulkan-specific - will consolidate into scripts/ like OpenXR soon 39e5c31af7Sopenharmony_cisys.path.insert(0, 'xml') 40e5c31af7Sopenharmony_ci 41e5c31af7Sopenharmony_cifrom apiconventions import APIConventions 42e5c31af7Sopenharmony_ciconventions = APIConventions() 43e5c31af7Sopenharmony_ci 44e5c31af7Sopenharmony_ci# Patterns used to recognize interesting lines in an asciidoc source file. 45e5c31af7Sopenharmony_ci# These patterns are only compiled once. 46e5c31af7Sopenharmony_ci 47e5c31af7Sopenharmony_ci# Find the pname: or code: patterns in a Valid Usage statement 48e5c31af7Sopenharmony_cipnamePat = re.compile(r'pname:(?P<param>\{?\w+\}?)') 49e5c31af7Sopenharmony_cicodePat = re.compile(r'code:(?P<param>\w+)') 50e5c31af7Sopenharmony_ci 51e5c31af7Sopenharmony_ci# Text that (may) not end sentences 52e5c31af7Sopenharmony_ci 53e5c31af7Sopenharmony_ci# A single letter followed by a period, typically a middle initial. 54e5c31af7Sopenharmony_ciendInitial = re.compile(r'^[A-Z]\.$') 55e5c31af7Sopenharmony_ci# An abbreviation, which does not (usually) end a line. 56e5c31af7Sopenharmony_ciendAbbrev = re.compile(r'(e\.g|i\.e|c\.f|vs)\.$', re.IGNORECASE) 57e5c31af7Sopenharmony_ci 58e5c31af7Sopenharmony_ci# Explicit Valid Usage list item with one or more leading asterisks 59e5c31af7Sopenharmony_ci# The re.DOTALL is needed to prevent vuPat.search() from stripping 60e5c31af7Sopenharmony_ci# the trailing newline. 61e5c31af7Sopenharmony_civuPat = re.compile(r'^(?P<head> [*]+)( *)(?P<tail>.*)', re.DOTALL) 62e5c31af7Sopenharmony_ci 63e5c31af7Sopenharmony_ci# VUID with the numeric portion captured in the match object 64e5c31af7Sopenharmony_civuidPat = re.compile(r'VUID-[^-]+-[^-]+-(?P<vuid>[0-9]+)') 65e5c31af7Sopenharmony_ci 66e5c31af7Sopenharmony_ci# Pattern matching leading nested bullet points 67e5c31af7Sopenharmony_ciglobal nestedVuPat 68e5c31af7Sopenharmony_cinestedVuPat = re.compile(r'^ \*\*') 69e5c31af7Sopenharmony_ci 70e5c31af7Sopenharmony_ciclass ReflowCallbacks: 71e5c31af7Sopenharmony_ci """State and arguments for reflowing. 72e5c31af7Sopenharmony_ci 73e5c31af7Sopenharmony_ci Used with DocTransformer to reflow a file.""" 74e5c31af7Sopenharmony_ci def __init__(self, 75e5c31af7Sopenharmony_ci filename, 76e5c31af7Sopenharmony_ci vuidDict, 77e5c31af7Sopenharmony_ci margin = 76, 78e5c31af7Sopenharmony_ci breakPeriod = True, 79e5c31af7Sopenharmony_ci reflow = True, 80e5c31af7Sopenharmony_ci nextvu = None, 81e5c31af7Sopenharmony_ci maxvu = None, 82e5c31af7Sopenharmony_ci check = True): 83e5c31af7Sopenharmony_ci 84e5c31af7Sopenharmony_ci self.filename = filename 85e5c31af7Sopenharmony_ci """base name of file being read from.""" 86e5c31af7Sopenharmony_ci 87e5c31af7Sopenharmony_ci self.check = check 88e5c31af7Sopenharmony_ci """Whether consistency checks must be performed.""" 89e5c31af7Sopenharmony_ci 90e5c31af7Sopenharmony_ci self.margin = margin 91e5c31af7Sopenharmony_ci """margin to reflow text to.""" 92e5c31af7Sopenharmony_ci 93e5c31af7Sopenharmony_ci self.breakPeriod = breakPeriod 94e5c31af7Sopenharmony_ci """True if justification should break to a new line after the end of a 95e5c31af7Sopenharmony_ci sentence.""" 96e5c31af7Sopenharmony_ci 97e5c31af7Sopenharmony_ci self.breakInitial = True 98e5c31af7Sopenharmony_ci """True if justification should break to a new line after something 99e5c31af7Sopenharmony_ci that appears to be an initial in someone's name. **TBD**""" 100e5c31af7Sopenharmony_ci 101e5c31af7Sopenharmony_ci self.reflow = reflow 102e5c31af7Sopenharmony_ci """True if text should be reflowed, False to pass through unchanged.""" 103e5c31af7Sopenharmony_ci 104e5c31af7Sopenharmony_ci self.vuPrefix = 'VUID' 105e5c31af7Sopenharmony_ci """Prefix of generated Valid Usage tags""" 106e5c31af7Sopenharmony_ci 107e5c31af7Sopenharmony_ci self.vuFormat = '{0}-{1}-{2}-{3:0>5d}' 108e5c31af7Sopenharmony_ci """Format string for generating Valid Usage tags. 109e5c31af7Sopenharmony_ci First argument is vuPrefix, second is command/struct name, third is 110e5c31af7Sopenharmony_ci parameter name, fourth is the tag number.""" 111e5c31af7Sopenharmony_ci 112e5c31af7Sopenharmony_ci self.nextvu = nextvu 113e5c31af7Sopenharmony_ci """Integer to start tagging un-numbered Valid Usage statements with, 114e5c31af7Sopenharmony_ci or None if no tagging should be done.""" 115e5c31af7Sopenharmony_ci 116e5c31af7Sopenharmony_ci self.maxvu = maxvu 117e5c31af7Sopenharmony_ci """Maximum tag to use for Valid Usage statements, or None if no 118e5c31af7Sopenharmony_ci tagging should be done.""" 119e5c31af7Sopenharmony_ci 120e5c31af7Sopenharmony_ci self.vuidDict = vuidDict 121e5c31af7Sopenharmony_ci """Dictionary of VUID numbers found, containing a list of (file, VUID) 122e5c31af7Sopenharmony_ci on which that number was found. This is used to warn on duplicate 123e5c31af7Sopenharmony_ci VUIDs.""" 124e5c31af7Sopenharmony_ci 125e5c31af7Sopenharmony_ci self.warnCount = 0 126e5c31af7Sopenharmony_ci """Count of markup check warnings encountered.""" 127e5c31af7Sopenharmony_ci 128e5c31af7Sopenharmony_ci def endSentence(self, word): 129e5c31af7Sopenharmony_ci """Return True if word ends with a sentence-period, False otherwise. 130e5c31af7Sopenharmony_ci 131e5c31af7Sopenharmony_ci Allows for contraction cases which will not end a line: 132e5c31af7Sopenharmony_ci 133e5c31af7Sopenharmony_ci - A single letter (if breakInitial is True) 134e5c31af7Sopenharmony_ci - Abbreviations: 'c.f.', 'e.g.', 'i.e.' (or mixed-case versions)""" 135e5c31af7Sopenharmony_ci if (word[-1:] != '.' or 136e5c31af7Sopenharmony_ci endAbbrev.search(word) or 137e5c31af7Sopenharmony_ci (self.breakInitial and endInitial.match(word))): 138e5c31af7Sopenharmony_ci return False 139e5c31af7Sopenharmony_ci 140e5c31af7Sopenharmony_ci return True 141e5c31af7Sopenharmony_ci 142e5c31af7Sopenharmony_ci def vuidAnchor(self, word): 143e5c31af7Sopenharmony_ci """Return True if word is a Valid Usage ID Tag anchor.""" 144e5c31af7Sopenharmony_ci return (word[0:7] == '[[VUID-') 145e5c31af7Sopenharmony_ci 146e5c31af7Sopenharmony_ci def visitVUID(self, vuid, line): 147e5c31af7Sopenharmony_ci if vuid not in self.vuidDict: 148e5c31af7Sopenharmony_ci self.vuidDict[vuid] = [] 149e5c31af7Sopenharmony_ci self.vuidDict[vuid].append([self.filename, line]) 150e5c31af7Sopenharmony_ci 151e5c31af7Sopenharmony_ci def gatherVUIDs(self, para): 152e5c31af7Sopenharmony_ci """Gather VUID tags and add them to vuidDict. Used to verify no-duplicate VUIDs""" 153e5c31af7Sopenharmony_ci for line in para: 154e5c31af7Sopenharmony_ci line = line.rstrip() 155e5c31af7Sopenharmony_ci 156e5c31af7Sopenharmony_ci matches = vuidPat.search(line) 157e5c31af7Sopenharmony_ci if matches is not None: 158e5c31af7Sopenharmony_ci vuid = matches.group('vuid') 159e5c31af7Sopenharmony_ci self.visitVUID(vuid, line) 160e5c31af7Sopenharmony_ci 161e5c31af7Sopenharmony_ci def addVUID(self, para, state): 162e5c31af7Sopenharmony_ci hangIndent = state.hangIndent 163e5c31af7Sopenharmony_ci 164e5c31af7Sopenharmony_ci """Generate and add VUID if necessary.""" 165e5c31af7Sopenharmony_ci if not state.isVU or self.nextvu is None: 166e5c31af7Sopenharmony_ci return para, hangIndent 167e5c31af7Sopenharmony_ci 168e5c31af7Sopenharmony_ci # If: 169e5c31af7Sopenharmony_ci # - this paragraph is in a Valid Usage block, 170e5c31af7Sopenharmony_ci # - VUID tags are being assigned, 171e5c31af7Sopenharmony_ci # Try to assign VUIDs 172e5c31af7Sopenharmony_ci 173e5c31af7Sopenharmony_ci if nestedVuPat.search(para[0]): 174e5c31af7Sopenharmony_ci # Do not assign VUIDs to nested bullet points. 175e5c31af7Sopenharmony_ci # These are now allowed VU markup syntax, but will never 176e5c31af7Sopenharmony_ci # themselves be VUs, just subsidiary points. 177e5c31af7Sopenharmony_ci return para, hangIndent 178e5c31af7Sopenharmony_ci 179e5c31af7Sopenharmony_ci # Skip if there is already a VUID assigned 180e5c31af7Sopenharmony_ci if self.vuPrefix in para[0]: 181e5c31af7Sopenharmony_ci return para, hangIndent 182e5c31af7Sopenharmony_ci 183e5c31af7Sopenharmony_ci # If: 184e5c31af7Sopenharmony_ci # - a tag is not already present, and 185e5c31af7Sopenharmony_ci # - the paragraph is a properly marked-up list item 186e5c31af7Sopenharmony_ci # Then add a VUID tag starting with the next free ID. 187e5c31af7Sopenharmony_ci 188e5c31af7Sopenharmony_ci # Split the first line after the bullet point 189e5c31af7Sopenharmony_ci matches = vuPat.search(para[0]) 190e5c31af7Sopenharmony_ci if matches is None: 191e5c31af7Sopenharmony_ci # There are only a few cases of this, and they are all 192e5c31af7Sopenharmony_ci # legitimate. Leave detecting this case to another tool 193e5c31af7Sopenharmony_ci # or hand inspection. 194e5c31af7Sopenharmony_ci # logWarn(self.filename + ': Unexpected non-bullet item in VU block (harmless if following an ifdef):', 195e5c31af7Sopenharmony_ci # para[0]) 196e5c31af7Sopenharmony_ci return para, hangIndent 197e5c31af7Sopenharmony_ci 198e5c31af7Sopenharmony_ci outPara = para 199e5c31af7Sopenharmony_ci 200e5c31af7Sopenharmony_ci logDiag('addVUID: Matched vuPat on line:', para[0], end='') 201e5c31af7Sopenharmony_ci head = matches.group('head') 202e5c31af7Sopenharmony_ci tail = matches.group('tail') 203e5c31af7Sopenharmony_ci 204e5c31af7Sopenharmony_ci # Find pname: or code: tags in the paragraph for the purposes of VUID 205e5c31af7Sopenharmony_ci # tag generation. pname:{attribute}s are prioritized to make sure 206e5c31af7Sopenharmony_ci # commonvalidity VUIDs end up being unique. Otherwise, the first pname: 207e5c31af7Sopenharmony_ci # or code: tag in the paragraph is used, which may not always be 208e5c31af7Sopenharmony_ci # correct, but should be highly reliable. 209e5c31af7Sopenharmony_ci pnameMatches = re.findall(pnamePat, ' '.join(para)) 210e5c31af7Sopenharmony_ci codeMatches = re.findall(codePat, ' '.join(para)) 211e5c31af7Sopenharmony_ci 212e5c31af7Sopenharmony_ci # Prioritize {attribute}s, but not the ones in the exception list 213e5c31af7Sopenharmony_ci # below. These have complex expressions including ., ->, or [index] 214e5c31af7Sopenharmony_ci # which makes them unsuitable for VUID tags. Ideally these would be 215e5c31af7Sopenharmony_ci # automatically discovered. 216e5c31af7Sopenharmony_ci attributeExceptionList = ['maxinstancecheck', 'regionsparam', 217e5c31af7Sopenharmony_ci 'rayGenShaderBindingTableAddress', 218e5c31af7Sopenharmony_ci 'rayGenShaderBindingTableStride', 219e5c31af7Sopenharmony_ci 'missShaderBindingTableAddress', 220e5c31af7Sopenharmony_ci 'missShaderBindingTableStride', 221e5c31af7Sopenharmony_ci 'hitShaderBindingTableAddress', 222e5c31af7Sopenharmony_ci 'hitShaderBindingTableStride', 223e5c31af7Sopenharmony_ci 'callableShaderBindingTableAddress', 224e5c31af7Sopenharmony_ci 'callableShaderBindingTableStride', 225e5c31af7Sopenharmony_ci ] 226e5c31af7Sopenharmony_ci attributeMatches = [match for match in pnameMatches if 227e5c31af7Sopenharmony_ci match[0] == '{' and 228e5c31af7Sopenharmony_ci match[1:-1] not in attributeExceptionList] 229e5c31af7Sopenharmony_ci nonattributeMatches = [match for match in pnameMatches if 230e5c31af7Sopenharmony_ci match[0] != '{'] 231e5c31af7Sopenharmony_ci 232e5c31af7Sopenharmony_ci if len(attributeMatches) > 0: 233e5c31af7Sopenharmony_ci paramName = attributeMatches[0] 234e5c31af7Sopenharmony_ci elif len(nonattributeMatches) > 0: 235e5c31af7Sopenharmony_ci paramName = nonattributeMatches[0] 236e5c31af7Sopenharmony_ci elif len(codeMatches) > 0: 237e5c31af7Sopenharmony_ci paramName = codeMatches[0] 238e5c31af7Sopenharmony_ci else: 239e5c31af7Sopenharmony_ci paramName = 'None' 240e5c31af7Sopenharmony_ci logWarn(self.filename, 241e5c31af7Sopenharmony_ci 'No param name found for VUID tag on line:', 242e5c31af7Sopenharmony_ci para[0]) 243e5c31af7Sopenharmony_ci 244e5c31af7Sopenharmony_ci # Transform: 245e5c31af7Sopenharmony_ci # 246e5c31af7Sopenharmony_ci # * VU first line 247e5c31af7Sopenharmony_ci # 248e5c31af7Sopenharmony_ci # To: 249e5c31af7Sopenharmony_ci # 250e5c31af7Sopenharmony_ci # * [[VUID]] 251e5c31af7Sopenharmony_ci # VU first line 252e5c31af7Sopenharmony_ci # 253e5c31af7Sopenharmony_ci tagLine = (head + ' [[' + 254e5c31af7Sopenharmony_ci self.vuFormat.format(self.vuPrefix, 255e5c31af7Sopenharmony_ci state.apiName, 256e5c31af7Sopenharmony_ci paramName, 257e5c31af7Sopenharmony_ci self.nextvu) + ']]\n') 258e5c31af7Sopenharmony_ci self.visitVUID(str(self.nextvu), tagLine) 259e5c31af7Sopenharmony_ci 260e5c31af7Sopenharmony_ci newLines = [tagLine] 261e5c31af7Sopenharmony_ci if tail.strip() != '': 262e5c31af7Sopenharmony_ci logDiag('transformParagraph first line matches bullet point -' 263e5c31af7Sopenharmony_ci 'single line, assuming hangIndent @ input line', 264e5c31af7Sopenharmony_ci state.lineNumber) 265e5c31af7Sopenharmony_ci hangIndent = len(head) + 1 266e5c31af7Sopenharmony_ci newLines.append(''.ljust(hangIndent) + tail) 267e5c31af7Sopenharmony_ci 268e5c31af7Sopenharmony_ci logDiag('Assigning', self.vuPrefix, state.apiName, self.nextvu, 269e5c31af7Sopenharmony_ci ' on line:\n' + para[0], '->\n' + newLines[0] + 'END', '\n' + newLines[1] if len(newLines) > 1 else '') 270e5c31af7Sopenharmony_ci 271e5c31af7Sopenharmony_ci # Do not actually assign the VUID unless it is in the reserved range 272e5c31af7Sopenharmony_ci if self.nextvu <= self.maxvu: 273e5c31af7Sopenharmony_ci if self.nextvu == self.maxvu: 274e5c31af7Sopenharmony_ci logWarn('Skipping VUID assignment, no more VUIDs available') 275e5c31af7Sopenharmony_ci outPara = newLines + para[1:] 276e5c31af7Sopenharmony_ci self.nextvu = self.nextvu + 1 277e5c31af7Sopenharmony_ci 278e5c31af7Sopenharmony_ci return outPara, hangIndent 279e5c31af7Sopenharmony_ci 280e5c31af7Sopenharmony_ci def transformParagraph(self, para, state): 281e5c31af7Sopenharmony_ci """Reflow a given paragraph, respecting the paragraph lead and 282e5c31af7Sopenharmony_ci hanging indentation levels. 283e5c31af7Sopenharmony_ci 284e5c31af7Sopenharmony_ci The algorithm also respects trailing '+' signs that indicate embedded newlines, 285e5c31af7Sopenharmony_ci and will not reflow a very long word immediately after a bullet point. 286e5c31af7Sopenharmony_ci 287e5c31af7Sopenharmony_ci Just return the paragraph unchanged if the -noflow argument was 288e5c31af7Sopenharmony_ci given.""" 289e5c31af7Sopenharmony_ci 290e5c31af7Sopenharmony_ci self.gatherVUIDs(para) 291e5c31af7Sopenharmony_ci 292e5c31af7Sopenharmony_ci # If this is a VU that is missing a VUID, add it to the paragraph now. 293e5c31af7Sopenharmony_ci para, hangIndent = self.addVUID(para, state) 294e5c31af7Sopenharmony_ci 295e5c31af7Sopenharmony_ci if not self.reflow: 296e5c31af7Sopenharmony_ci return para 297e5c31af7Sopenharmony_ci 298e5c31af7Sopenharmony_ci logDiag('transformParagraph lead indent = ', state.leadIndent, 299e5c31af7Sopenharmony_ci 'hangIndent =', state.hangIndent, 300e5c31af7Sopenharmony_ci 'para:', para[0], end='') 301e5c31af7Sopenharmony_ci 302e5c31af7Sopenharmony_ci # Total words processed (we care about the *first* word vs. others) 303e5c31af7Sopenharmony_ci wordCount = 0 304e5c31af7Sopenharmony_ci 305e5c31af7Sopenharmony_ci # Tracks the *previous* word processed. It must not be empty. 306e5c31af7Sopenharmony_ci prevWord = ' ' 307e5c31af7Sopenharmony_ci 308e5c31af7Sopenharmony_ci # Track the previous line and paragraph being indented, if any 309e5c31af7Sopenharmony_ci outLine = None 310e5c31af7Sopenharmony_ci outPara = [] 311e5c31af7Sopenharmony_ci 312e5c31af7Sopenharmony_ci for line in para: 313e5c31af7Sopenharmony_ci line = line.rstrip() 314e5c31af7Sopenharmony_ci words = line.split() 315e5c31af7Sopenharmony_ci 316e5c31af7Sopenharmony_ci # logDiag('transformParagraph: input line =', line) 317e5c31af7Sopenharmony_ci numWords = len(words) - 1 318e5c31af7Sopenharmony_ci 319e5c31af7Sopenharmony_ci for i in range(0, numWords + 1): 320e5c31af7Sopenharmony_ci word = words[i] 321e5c31af7Sopenharmony_ci wordLen = len(word) 322e5c31af7Sopenharmony_ci wordCount += 1 323e5c31af7Sopenharmony_ci 324e5c31af7Sopenharmony_ci endEscape = False 325e5c31af7Sopenharmony_ci if i == numWords and word in ('+', '-'): 326e5c31af7Sopenharmony_ci # Trailing ' +' or ' -' must stay on the same line 327e5c31af7Sopenharmony_ci endEscape = word 328e5c31af7Sopenharmony_ci # logDiag('transformParagraph last word of line =', word, 329e5c31af7Sopenharmony_ci # 'prevWord =', prevWord, 'endEscape =', endEscape) 330e5c31af7Sopenharmony_ci else: 331e5c31af7Sopenharmony_ci # logDiag('transformParagraph wordCount =', wordCount, 332e5c31af7Sopenharmony_ci # 'word =', word, 'prevWord =', prevWord) 333e5c31af7Sopenharmony_ci pass 334e5c31af7Sopenharmony_ci 335e5c31af7Sopenharmony_ci if wordCount == 1: 336e5c31af7Sopenharmony_ci # The first word of the paragraph is treated specially. 337e5c31af7Sopenharmony_ci # The loop logic becomes trickier if all this code is 338e5c31af7Sopenharmony_ci # done prior to looping over lines and words, so all the 339e5c31af7Sopenharmony_ci # setup logic is done here. 340e5c31af7Sopenharmony_ci 341e5c31af7Sopenharmony_ci outLine = ''.ljust(state.leadIndent) + word 342e5c31af7Sopenharmony_ci outLineLen = state.leadIndent + wordLen 343e5c31af7Sopenharmony_ci 344e5c31af7Sopenharmony_ci # If the paragraph begins with a bullet point, generate 345e5c31af7Sopenharmony_ci # a hanging indent level if there is not one already. 346e5c31af7Sopenharmony_ci if doctransformer.beginBullet.match(para[0]): 347e5c31af7Sopenharmony_ci bulletPoint = True 348e5c31af7Sopenharmony_ci if len(para) > 1: 349e5c31af7Sopenharmony_ci logDiag('transformParagraph first line matches bullet point', 350e5c31af7Sopenharmony_ci 'but indent already hanging @ input line', 351e5c31af7Sopenharmony_ci state.lineNumber) 352e5c31af7Sopenharmony_ci else: 353e5c31af7Sopenharmony_ci logDiag('transformParagraph first line matches bullet point -' 354e5c31af7Sopenharmony_ci 'single line, assuming hangIndent @ input line', 355e5c31af7Sopenharmony_ci state.lineNumber) 356e5c31af7Sopenharmony_ci hangIndent = outLineLen + 1 357e5c31af7Sopenharmony_ci else: 358e5c31af7Sopenharmony_ci bulletPoint = False 359e5c31af7Sopenharmony_ci else: 360e5c31af7Sopenharmony_ci # Possible actions to take with this word 361e5c31af7Sopenharmony_ci # 362e5c31af7Sopenharmony_ci # addWord - add word to current line 363e5c31af7Sopenharmony_ci # closeLine - append line and start a new (null) one 364e5c31af7Sopenharmony_ci # startLine - add word to a new line 365e5c31af7Sopenharmony_ci 366e5c31af7Sopenharmony_ci # Default behavior if all the tests below fail is to add 367e5c31af7Sopenharmony_ci # this word to the current line, and keep accumulating 368e5c31af7Sopenharmony_ci # that line. 369e5c31af7Sopenharmony_ci (addWord, closeLine, startLine) = (True, False, False) 370e5c31af7Sopenharmony_ci 371e5c31af7Sopenharmony_ci # How long would this line be if the word were added? 372e5c31af7Sopenharmony_ci newLen = outLineLen + 1 + wordLen 373e5c31af7Sopenharmony_ci 374e5c31af7Sopenharmony_ci # Are we on the first word following a bullet point? 375e5c31af7Sopenharmony_ci firstBullet = (wordCount == 2 and bulletPoint) 376e5c31af7Sopenharmony_ci 377e5c31af7Sopenharmony_ci if endEscape: 378e5c31af7Sopenharmony_ci # If the new word ends the input line with ' +', 379e5c31af7Sopenharmony_ci # add it to the current line. 380e5c31af7Sopenharmony_ci 381e5c31af7Sopenharmony_ci (addWord, closeLine, startLine) = (True, True, False) 382e5c31af7Sopenharmony_ci elif self.vuidAnchor(word): 383e5c31af7Sopenharmony_ci # If the new word is a Valid Usage anchor, break the 384e5c31af7Sopenharmony_ci # line afterwards. Note that this should only happen 385e5c31af7Sopenharmony_ci # immediately after a bullet point, but we do not 386e5c31af7Sopenharmony_ci # currently check for this. 387e5c31af7Sopenharmony_ci (addWord, closeLine, startLine) = (True, True, False) 388e5c31af7Sopenharmony_ci 389e5c31af7Sopenharmony_ci elif newLen > self.margin: 390e5c31af7Sopenharmony_ci if firstBullet: 391e5c31af7Sopenharmony_ci # If the word follows a bullet point, add it to 392e5c31af7Sopenharmony_ci # the current line no matter its length. 393e5c31af7Sopenharmony_ci 394e5c31af7Sopenharmony_ci (addWord, closeLine, startLine) = (True, True, False) 395e5c31af7Sopenharmony_ci elif doctransformer.beginBullet.match(word + ' '): 396e5c31af7Sopenharmony_ci # If the word *is* a bullet point, add it to 397e5c31af7Sopenharmony_ci # the current line no matter its length. 398e5c31af7Sopenharmony_ci # This avoids an innocent inline '-' or '*' 399e5c31af7Sopenharmony_ci # turning into a bogus bullet point. 400e5c31af7Sopenharmony_ci 401e5c31af7Sopenharmony_ci (addWord, closeLine, startLine) = (True, True, False) 402e5c31af7Sopenharmony_ci else: 403e5c31af7Sopenharmony_ci # The word overflows, so add it to a new line. 404e5c31af7Sopenharmony_ci 405e5c31af7Sopenharmony_ci (addWord, closeLine, startLine) = (False, True, True) 406e5c31af7Sopenharmony_ci elif (self.breakPeriod and 407e5c31af7Sopenharmony_ci (wordCount > 2 or not firstBullet) and 408e5c31af7Sopenharmony_ci self.endSentence(prevWord)): 409e5c31af7Sopenharmony_ci # If the previous word ends a sentence and 410e5c31af7Sopenharmony_ci # breakPeriod is set, start a new line. 411e5c31af7Sopenharmony_ci # The complicated logic allows for leading bullet 412e5c31af7Sopenharmony_ci # points which are periods (implicitly numbered lists). 413e5c31af7Sopenharmony_ci # @@@ But not yet for explicitly numbered lists. 414e5c31af7Sopenharmony_ci 415e5c31af7Sopenharmony_ci (addWord, closeLine, startLine) = (False, True, True) 416e5c31af7Sopenharmony_ci 417e5c31af7Sopenharmony_ci # Add a word to the current line 418e5c31af7Sopenharmony_ci if addWord: 419e5c31af7Sopenharmony_ci if outLine: 420e5c31af7Sopenharmony_ci outLine += ' ' + word 421e5c31af7Sopenharmony_ci outLineLen = newLen 422e5c31af7Sopenharmony_ci else: 423e5c31af7Sopenharmony_ci # Fall through to startLine case if there is no 424e5c31af7Sopenharmony_ci # current line yet. 425e5c31af7Sopenharmony_ci startLine = True 426e5c31af7Sopenharmony_ci 427e5c31af7Sopenharmony_ci # Add current line to the output paragraph. Force 428e5c31af7Sopenharmony_ci # starting a new line, although we do not yet know if it 429e5c31af7Sopenharmony_ci # will ever have contents. 430e5c31af7Sopenharmony_ci if closeLine: 431e5c31af7Sopenharmony_ci if outLine: 432e5c31af7Sopenharmony_ci outPara.append(outLine + '\n') 433e5c31af7Sopenharmony_ci outLine = None 434e5c31af7Sopenharmony_ci 435e5c31af7Sopenharmony_ci # Start a new line and add a word to it 436e5c31af7Sopenharmony_ci if startLine: 437e5c31af7Sopenharmony_ci outLine = ''.ljust(hangIndent) + word 438e5c31af7Sopenharmony_ci outLineLen = hangIndent + wordLen 439e5c31af7Sopenharmony_ci 440e5c31af7Sopenharmony_ci # Track the previous word, for use in breaking at end of 441e5c31af7Sopenharmony_ci # a sentence 442e5c31af7Sopenharmony_ci prevWord = word 443e5c31af7Sopenharmony_ci 444e5c31af7Sopenharmony_ci # Add last line to the output paragraph. 445e5c31af7Sopenharmony_ci if outLine: 446e5c31af7Sopenharmony_ci outPara.append(outLine + '\n') 447e5c31af7Sopenharmony_ci 448e5c31af7Sopenharmony_ci return outPara 449e5c31af7Sopenharmony_ci 450e5c31af7Sopenharmony_ci def onEmbeddedVUConditional(self, state): 451e5c31af7Sopenharmony_ci if self.check: 452e5c31af7Sopenharmony_ci logWarn('Detected embedded Valid Usage conditional: {}:{}'.format( 453e5c31af7Sopenharmony_ci self.filename, state.lineNumber - 1)) 454e5c31af7Sopenharmony_ci # Keep track of warning check count 455e5c31af7Sopenharmony_ci self.warnCount = self.warnCount + 1 456e5c31af7Sopenharmony_ci 457e5c31af7Sopenharmony_cidef reflowFile(filename, args): 458e5c31af7Sopenharmony_ci logDiag('reflow: filename', filename) 459e5c31af7Sopenharmony_ci 460e5c31af7Sopenharmony_ci # Output file handle and reflow object for this file. There are no race 461e5c31af7Sopenharmony_ci # conditions on overwriting the input, but it is not recommended unless 462e5c31af7Sopenharmony_ci # you have backing store such as git. 463e5c31af7Sopenharmony_ci 464e5c31af7Sopenharmony_ci lines, newline_string = loadFile(filename) 465e5c31af7Sopenharmony_ci if lines is None: 466e5c31af7Sopenharmony_ci return 467e5c31af7Sopenharmony_ci 468e5c31af7Sopenharmony_ci if args.overwrite: 469e5c31af7Sopenharmony_ci outFilename = filename 470e5c31af7Sopenharmony_ci else: 471e5c31af7Sopenharmony_ci outDir = Path(args.outDir).resolve() 472e5c31af7Sopenharmony_ci outDir.mkdir(parents=True, exist_ok=True) 473e5c31af7Sopenharmony_ci 474e5c31af7Sopenharmony_ci outFilename = str(outDir / (os.path.basename(filename) + args.suffix)) 475e5c31af7Sopenharmony_ci 476e5c31af7Sopenharmony_ci if args.nowrite: 477e5c31af7Sopenharmony_ci fp = None 478e5c31af7Sopenharmony_ci else: 479e5c31af7Sopenharmony_ci try: 480e5c31af7Sopenharmony_ci fp = open(outFilename, 'w', encoding='utf8', newline=newline_string) 481e5c31af7Sopenharmony_ci except: 482e5c31af7Sopenharmony_ci logWarn('Cannot open output file', outFilename, ':', sys.exc_info()[0]) 483e5c31af7Sopenharmony_ci return 484e5c31af7Sopenharmony_ci 485e5c31af7Sopenharmony_ci callback = ReflowCallbacks(filename, 486e5c31af7Sopenharmony_ci args.vuidDict, 487e5c31af7Sopenharmony_ci margin = args.margin, 488e5c31af7Sopenharmony_ci reflow = not args.noflow, 489e5c31af7Sopenharmony_ci nextvu = args.nextvu, 490e5c31af7Sopenharmony_ci maxvu = args.maxvu, 491e5c31af7Sopenharmony_ci check = args.check) 492e5c31af7Sopenharmony_ci 493e5c31af7Sopenharmony_ci transformer = doctransformer.DocTransformer(filename, 494e5c31af7Sopenharmony_ci outfile = fp, 495e5c31af7Sopenharmony_ci callback = callback) 496e5c31af7Sopenharmony_ci 497e5c31af7Sopenharmony_ci transformer.transformFile(lines) 498e5c31af7Sopenharmony_ci 499e5c31af7Sopenharmony_ci if fp is not None: 500e5c31af7Sopenharmony_ci fp.close() 501e5c31af7Sopenharmony_ci 502e5c31af7Sopenharmony_ci # Update the 'nextvu' value 503e5c31af7Sopenharmony_ci if args.nextvu != callback.nextvu: 504e5c31af7Sopenharmony_ci logWarn('Updated nextvu to', callback.nextvu, 'after file', filename) 505e5c31af7Sopenharmony_ci args.nextvu = callback.nextvu 506e5c31af7Sopenharmony_ci 507e5c31af7Sopenharmony_ci args.warnCount += callback.warnCount 508e5c31af7Sopenharmony_ci 509e5c31af7Sopenharmony_cidef reflowAllAdocFiles(folder_to_reflow, args): 510e5c31af7Sopenharmony_ci for root, subdirs, files in os.walk(folder_to_reflow): 511e5c31af7Sopenharmony_ci for file in files: 512e5c31af7Sopenharmony_ci if file.endswith(conventions.file_suffix): 513e5c31af7Sopenharmony_ci file_path = os.path.join(root, file) 514e5c31af7Sopenharmony_ci reflowFile(file_path, args) 515e5c31af7Sopenharmony_ci for subdir in subdirs: 516e5c31af7Sopenharmony_ci sub_folder = os.path.join(root, subdir) 517e5c31af7Sopenharmony_ci print('Sub-folder = %s' % sub_folder) 518e5c31af7Sopenharmony_ci if subdir.lower() not in conventions.spec_no_reflow_dirs: 519e5c31af7Sopenharmony_ci print(' Parsing = %s' % sub_folder) 520e5c31af7Sopenharmony_ci reflowAllAdocFiles(sub_folder, args) 521e5c31af7Sopenharmony_ci else: 522e5c31af7Sopenharmony_ci print(' Skipping = %s' % sub_folder) 523e5c31af7Sopenharmony_ci 524e5c31af7Sopenharmony_ciif __name__ == '__main__': 525e5c31af7Sopenharmony_ci parser = argparse.ArgumentParser() 526e5c31af7Sopenharmony_ci 527e5c31af7Sopenharmony_ci parser.add_argument('-diag', action='store', dest='diagFile', 528e5c31af7Sopenharmony_ci help='Set the diagnostic file') 529e5c31af7Sopenharmony_ci parser.add_argument('-warn', action='store', dest='warnFile', 530e5c31af7Sopenharmony_ci help='Set the warning file') 531e5c31af7Sopenharmony_ci parser.add_argument('-log', action='store', dest='logFile', 532e5c31af7Sopenharmony_ci help='Set the log file for both diagnostics and warnings') 533e5c31af7Sopenharmony_ci parser.add_argument('-overwrite', action='store_true', 534e5c31af7Sopenharmony_ci help='Overwrite input filenames instead of writing different output filenames') 535e5c31af7Sopenharmony_ci parser.add_argument('-out', action='store', dest='outDir', 536e5c31af7Sopenharmony_ci default='out', 537e5c31af7Sopenharmony_ci help='Set the output directory in which updated files are generated (default: out)') 538e5c31af7Sopenharmony_ci parser.add_argument('-nowrite', action='store_true', 539e5c31af7Sopenharmony_ci help='Do not write output files, for use with -check') 540e5c31af7Sopenharmony_ci parser.add_argument('-check', action='store', dest='check', 541e5c31af7Sopenharmony_ci help='Run markup checks and warn if WARN option is given, error exit if FAIL option is given') 542e5c31af7Sopenharmony_ci parser.add_argument('-checkVUID', action='store', dest='checkVUID', 543e5c31af7Sopenharmony_ci help='Detect duplicated VUID numbers and warn if WARN option is given, error exit if FAIL option is given') 544e5c31af7Sopenharmony_ci parser.add_argument('-tagvu', action='store_true', 545e5c31af7Sopenharmony_ci help='Tag un-tagged Valid Usage statements starting at the value wired into reflow.py') 546e5c31af7Sopenharmony_ci parser.add_argument('-nextvu', action='store', dest='nextvu', type=int, 547e5c31af7Sopenharmony_ci default=None, 548e5c31af7Sopenharmony_ci help='Tag un-tagged Valid Usage statements starting at the specified base VUID instead of the value wired into reflow.py') 549e5c31af7Sopenharmony_ci parser.add_argument('-maxvu', action='store', dest='maxvu', type=int, 550e5c31af7Sopenharmony_ci default=None, 551e5c31af7Sopenharmony_ci help='Specify maximum VUID instead of the value wired into vuidCounts.py') 552e5c31af7Sopenharmony_ci parser.add_argument('-branch', action='store', dest='branch', 553e5c31af7Sopenharmony_ci help='Specify branch to assign VUIDs for') 554e5c31af7Sopenharmony_ci parser.add_argument('-noflow', action='store_true', dest='noflow', 555e5c31af7Sopenharmony_ci help='Do not reflow text. Other actions may apply') 556e5c31af7Sopenharmony_ci parser.add_argument('-margin', action='store', type=int, dest='margin', 557e5c31af7Sopenharmony_ci default='76', 558e5c31af7Sopenharmony_ci help='Width to reflow text, defaults to 76 characters') 559e5c31af7Sopenharmony_ci parser.add_argument('-suffix', action='store', dest='suffix', 560e5c31af7Sopenharmony_ci default='', 561e5c31af7Sopenharmony_ci help='Set the suffix added to updated file names (default: none)') 562e5c31af7Sopenharmony_ci parser.add_argument('files', metavar='filename', nargs='*', 563e5c31af7Sopenharmony_ci help='a filename to reflow text in') 564e5c31af7Sopenharmony_ci parser.add_argument('--version', action='version', version='%(prog)s 1.0') 565e5c31af7Sopenharmony_ci 566e5c31af7Sopenharmony_ci args = parser.parse_args() 567e5c31af7Sopenharmony_ci 568e5c31af7Sopenharmony_ci setLogFile(True, True, args.logFile) 569e5c31af7Sopenharmony_ci setLogFile(True, False, args.diagFile) 570e5c31af7Sopenharmony_ci setLogFile(False, True, args.warnFile) 571e5c31af7Sopenharmony_ci 572e5c31af7Sopenharmony_ci if args.overwrite: 573e5c31af7Sopenharmony_ci logWarn("reflow.py: will overwrite all input files") 574e5c31af7Sopenharmony_ci 575e5c31af7Sopenharmony_ci errors = '' 576e5c31af7Sopenharmony_ci if args.branch is None: 577e5c31af7Sopenharmony_ci (args.branch, errors) = getBranch() 578e5c31af7Sopenharmony_ci if args.branch is None: 579e5c31af7Sopenharmony_ci # This is not fatal unless VUID assignment is required 580e5c31af7Sopenharmony_ci if args.tagvu: 581e5c31af7Sopenharmony_ci logErr('Cannot determine current git branch, so cannot assign VUIDs:', errors) 582e5c31af7Sopenharmony_ci 583e5c31af7Sopenharmony_ci if args.tagvu and args.nextvu is None: 584e5c31af7Sopenharmony_ci # Moved here since vuidCounts is only needed in the internal 585e5c31af7Sopenharmony_ci # repository 586e5c31af7Sopenharmony_ci from vuidCounts import vuidCounts 587e5c31af7Sopenharmony_ci 588e5c31af7Sopenharmony_ci if args.branch not in vuidCounts: 589e5c31af7Sopenharmony_ci logErr('Branch', args.branch, 'not in vuidCounts, cannot continue') 590e5c31af7Sopenharmony_ci maxVUID = vuidCounts[args.branch][1] 591e5c31af7Sopenharmony_ci startVUID = vuidCounts[args.branch][2] 592e5c31af7Sopenharmony_ci args.nextvu = startVUID 593e5c31af7Sopenharmony_ci args.maxvu = maxVUID 594e5c31af7Sopenharmony_ci 595e5c31af7Sopenharmony_ci if args.nextvu is not None: 596e5c31af7Sopenharmony_ci logWarn('Tagging untagged Valid Usage statements starting at', args.nextvu) 597e5c31af7Sopenharmony_ci 598e5c31af7Sopenharmony_ci # Count of markup check warnings encountered 599e5c31af7Sopenharmony_ci # This is added to the argparse structure 600e5c31af7Sopenharmony_ci args.warnCount = 0 601e5c31af7Sopenharmony_ci 602e5c31af7Sopenharmony_ci # Dictionary of VUID numbers found, containing a list of (file, VUID) on 603e5c31af7Sopenharmony_ci # which that number was found 604e5c31af7Sopenharmony_ci # This is added to the argparse structure 605e5c31af7Sopenharmony_ci args.vuidDict = {} 606e5c31af7Sopenharmony_ci 607e5c31af7Sopenharmony_ci # If no files are specified, reflow the entire specification chapters folder 608e5c31af7Sopenharmony_ci if not args.files: 609e5c31af7Sopenharmony_ci folder_to_reflow = conventions.spec_reflow_path 610e5c31af7Sopenharmony_ci logWarn('Reflowing all asciidoc files under', folder_to_reflow) 611e5c31af7Sopenharmony_ci reflowAllAdocFiles(folder_to_reflow, args) 612e5c31af7Sopenharmony_ci else: 613e5c31af7Sopenharmony_ci for file in args.files: 614e5c31af7Sopenharmony_ci reflowFile(file, args) 615e5c31af7Sopenharmony_ci 616e5c31af7Sopenharmony_ci if args.warnCount > 0: 617e5c31af7Sopenharmony_ci if args.check == 'FAIL': 618e5c31af7Sopenharmony_ci logErr('Failed with', args.warnCount, 'markup errors detected.\n' + 619e5c31af7Sopenharmony_ci 'To fix these, you can take actions such as:\n' + 620e5c31af7Sopenharmony_ci ' * Moving conditionals outside VU start / end without changing VU meaning\n' + 621e5c31af7Sopenharmony_ci ' * Refactor conditional text using terminology defined conditionally outside the VU itself\n' + 622e5c31af7Sopenharmony_ci ' * Remove the conditional (allowable when this just affects command / structure / enum names)\n') 623e5c31af7Sopenharmony_ci else: 624e5c31af7Sopenharmony_ci logWarn('Total warning count for markup issues is', args.warnCount) 625e5c31af7Sopenharmony_ci 626e5c31af7Sopenharmony_ci # Look for duplicated VUID numbers 627e5c31af7Sopenharmony_ci if args.checkVUID: 628e5c31af7Sopenharmony_ci dupVUIDs = 0 629e5c31af7Sopenharmony_ci for vuid in sorted(args.vuidDict): 630e5c31af7Sopenharmony_ci found = args.vuidDict[vuid] 631e5c31af7Sopenharmony_ci if len(found) > 1: 632e5c31af7Sopenharmony_ci logWarn('Duplicate VUID number {} found in files:'.format(vuid)) 633e5c31af7Sopenharmony_ci for (file, vuidLine) in found: 634e5c31af7Sopenharmony_ci logWarn(' {}: {}'.format(file, vuidLine)) 635e5c31af7Sopenharmony_ci dupVUIDs = dupVUIDs + 1 636e5c31af7Sopenharmony_ci 637e5c31af7Sopenharmony_ci if dupVUIDs > 0: 638e5c31af7Sopenharmony_ci if args.checkVUID == 'FAIL': 639e5c31af7Sopenharmony_ci logErr('Failed with', dupVUIDs, 'duplicated VUID numbers found.\n' + 640e5c31af7Sopenharmony_ci 'To fix this, either convert these to commonvalidity VUs if possible, or strip\n' + 641e5c31af7Sopenharmony_ci 'the VUIDs from all but one of the duplicates and regenerate new ones.') 642e5c31af7Sopenharmony_ci else: 643e5c31af7Sopenharmony_ci logWarn('Total number of duplicated VUID numbers is', dupVUIDs) 644e5c31af7Sopenharmony_ci 645e5c31af7Sopenharmony_ci if args.nextvu is not None and args.nextvu != startVUID: 646e5c31af7Sopenharmony_ci # Update next free VUID to assign 647e5c31af7Sopenharmony_ci vuidCounts[args.branch][2] = args.nextvu 648e5c31af7Sopenharmony_ci try: 649e5c31af7Sopenharmony_ci reflow_count_file_path = os.path.dirname(os.path.realpath(__file__)) 650e5c31af7Sopenharmony_ci reflow_count_file_path += '/vuidCounts.py' 651e5c31af7Sopenharmony_ci reflow_count_file = open(reflow_count_file_path, 'w', encoding='utf8') 652e5c31af7Sopenharmony_ci print('# Do not edit this file, unless reserving a new VUID range', file=reflow_count_file) 653e5c31af7Sopenharmony_ci print('# VUID ranges reserved for branches', file=reflow_count_file) 654e5c31af7Sopenharmony_ci print('# Key is branch name, value is [ start, end, nextfree ]', file=reflow_count_file) 655e5c31af7Sopenharmony_ci print('# New reservations must be made by MR to main branch', file=reflow_count_file) 656e5c31af7Sopenharmony_ci print('vuidCounts = {', file=reflow_count_file) 657e5c31af7Sopenharmony_ci for key in sorted(vuidCounts.keys(), key=lambda k: vuidCounts[k][0]): 658e5c31af7Sopenharmony_ci counts = vuidCounts[key] 659e5c31af7Sopenharmony_ci print(f" '{key}': [ {counts[0]}, {counts[1]}, {counts[2]} ],", 660e5c31af7Sopenharmony_ci file=reflow_count_file) 661e5c31af7Sopenharmony_ci print('}', file=reflow_count_file) 662e5c31af7Sopenharmony_ci reflow_count_file.close() 663e5c31af7Sopenharmony_ci except: 664e5c31af7Sopenharmony_ci logWarn('Cannot open output count file vuidCounts.py', ':', sys.exc_info()[0]) 665