1/*
2 * Copyright (c) 2021-2022 Huawei Device Co., Ltd.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at
6 *
7 * http://www.apache.org/licenses/LICENSE-2.0
8 *
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
14 */
15
16#include "compileQueue.h"
17
18#include <mem/arena_allocator.h>
19#include <mem/pool_manager.h>
20#include "utils/timers.h"
21
22#include <binder/binder.h>
23#include <binder/scope.h>
24#include <compiler/core/compilerContext.h>
25#include <compiler/core/emitter/emitter.h>
26#include <compiler/core/function.h>
27#include <compiler/core/pandagen.h>
28#include <es2panda.h>
29#include <protobufSnapshotGenerator.h>
30#include <util/commonUtil.h>
31#include <util/dumper.h>
32#include <util/helpers.h>
33
34namespace panda::es2panda::compiler {
35
36std::mutex CompileFileJob::globalMutex_;
37std::mutex CompileAbcClassQueue::globalMutex_;
38
39void CompileFunctionJob::Run()
40{
41    std::unique_lock<std::mutex> lock(m_);
42    cond_.wait(lock, [this] { return dependencies_ == 0; });
43
44    ArenaAllocator allocator(SpaceType::SPACE_TYPE_COMPILER, nullptr, true);
45    PandaGen pg(&allocator, context_, scope_);
46
47    Function::Compile(&pg);
48
49    FunctionEmitter funcEmitter(&allocator, &pg);
50    funcEmitter.Generate(context_->PatchFixHelper());
51
52    context_->GetEmitter()->AddFunction(&funcEmitter, context_);
53
54    for (auto *dependant : dependants_) {
55        dependant->Signal();
56    }
57}
58
59void CompileModuleRecordJob::Run()
60{
61    std::unique_lock<std::mutex> lock(m_);
62    cond_.wait(lock, [this] { return dependencies_ == 0; });
63
64    bool hasLazyImport = context_->Binder()->Program()->ModuleRecord()->HasLazyImport();
65    ModuleRecordEmitter moduleEmitter(context_->Binder()->Program()->ModuleRecord(), context_->NewLiteralIndex(),
66        hasLazyImport ? context_->NewLiteralIndex() : -1);
67    moduleEmitter.Generate();
68
69    context_->GetEmitter()->AddSourceTextModuleRecord(&moduleEmitter, context_);
70
71    for (auto *dependant : dependants_) {
72        dependant->Signal();
73    }
74}
75
76bool CompileFileJob::RetrieveProgramFromCacheFiles(const std::string &buffer)
77{
78    if (options_->requireGlobalOptimization) {
79        return false;
80    }
81    auto cacheFileIter = options_->cacheFiles.find(src_->fileName);
82    // Disable the use of file caching when cross-program optimization is required, to prevent cached files from
83    // not being invalidated when their dependencies change, or from not being reanalyzed when their dependents
84    // are updated
85    if (cacheFileIter != options_->cacheFiles.end()) {
86        // cache is invalid when any one of source file infos being changed
87        auto bufToHash = buffer + src_->fileName + src_->recordName + src_->sourcefile + src_->pkgName;
88        src_->hash = GetHash32String(reinterpret_cast<const uint8_t *>(bufToHash.c_str()));
89
90        ArenaAllocator allocator(SpaceType::SPACE_TYPE_COMPILER, nullptr, true);
91        auto *cacheProgramInfo = proto::ProtobufSnapshotGenerator::GetCacheContext(cacheFileIter->second,
92                                                                                   &allocator);
93
94        if (cacheProgramInfo != nullptr && cacheProgramInfo->hashCode == src_->hash) {
95            std::unique_lock<std::mutex> lock(globalMutex_);
96            auto *cache = allocator_->New<util::ProgramCache>(src_->hash, std::move(cacheProgramInfo->program));
97            progsInfo_.insert({src_->fileName, cache});
98            return true;
99        }
100    }
101    return false;
102}
103
104void CompileFileJob::Run()
105{
106    std::stringstream ss;
107    std::string buffer;
108    panda::Timer::timerStart(panda::EVENT_READ_INPUT_AND_CACHE, src_->fileName);
109    if (!src_->fileName.empty() && src_->isSourceMode) {
110        if (!util::Helpers::ReadFileToBuffer(src_->fileName, ss)) {
111            return;
112        }
113        buffer = ss.str();
114        src_->source = buffer;
115        if (RetrieveProgramFromCacheFiles(buffer)) {
116            panda::Timer::timerEnd(panda::EVENT_READ_INPUT_AND_CACHE, src_->fileName);
117            return;
118        }
119    }
120    panda::Timer::timerEnd(panda::EVENT_READ_INPUT_AND_CACHE, src_->fileName);
121
122    CompileProgram();
123}
124
125void CompileFileJob::CompileProgram()
126{
127    es2panda::Compiler compiler(src_->scriptExtension, options_->functionThreadCount);
128    panda::pandasm::Program *prog = nullptr;
129
130    if (src_->isSourceMode) {
131        panda::Timer::timerStart(panda::EVENT_COMPILE_FILE, src_->fileName);
132        prog = compiler.CompileFile(*options_, src_, symbolTable_);
133        panda::Timer::timerEnd(panda::EVENT_COMPILE_FILE, src_->fileName);
134    } else if (!options_->mergeAbc) {
135        // If input is an abc file, in non merge-abc mode, compile classes one by one.
136        panda::Timer::timerStart(panda::EVENT_COMPILE_ABC_FILE, src_->fileName);
137        prog = compiler.CompileAbcFile(src_->fileName, *options_);
138        panda::Timer::timerEnd(panda::EVENT_COMPILE_ABC_FILE, src_->fileName);
139    } else {
140        // If input is an abc file, in merge-abc mode, compile each class parallelly.
141        panda::Timer::timerStart(panda::EVENT_COMPILE_ABC_FILE, src_->fileName);
142        compiler.CompileAbcFileInParallel(src_, *options_, progsInfo_, allocator_);
143        panda::Timer::timerEnd(panda::EVENT_COMPILE_ABC_FILE, src_->fileName);
144        return;
145    }
146
147    if (prog == nullptr) {
148        return;
149    }
150
151    OptimizeAndCacheProgram(prog);
152}
153
154void CompileFileJob::OptimizeAndCacheProgram(panda::pandasm::Program *prog)
155{
156    bool requireOptimizationAfterAnalysis = false;
157    // When cross-program optimizations are required, skip program-local optimization at this stage
158    // and perform it later after the analysis of all programs has been completed
159    if (src_->isSourceMode && options_->transformLib.empty()) {
160        if (options_->requireGlobalOptimization) {
161            panda::Timer::timerStart(panda::EVENT_OPTIMIZE_PROGRAM, src_->fileName);
162            util::Helpers::AnalysisProgram(prog, src_->fileName);
163            requireOptimizationAfterAnalysis = true;
164        } else if (options_->optLevel != 0) {
165            panda::Timer::timerStart(panda::EVENT_OPTIMIZE_PROGRAM, src_->fileName);
166            util::Helpers::OptimizeProgram(prog, src_->fileName);
167            panda::Timer::timerEnd(panda::EVENT_OPTIMIZE_PROGRAM, src_->fileName);
168        }
169    }
170
171    {
172        std::unique_lock<std::mutex> lock(globalMutex_);
173        auto *cache = allocator_->New<util::ProgramCache>(src_->hash, std::move(*prog), src_->isSourceMode);
174        progsInfo_.insert({src_->fileName, cache});
175        if (requireOptimizationAfterAnalysis) {
176            optimizationPendingProgs_.insert(src_->fileName);
177        }
178    }
179}
180
181void CompileAbcClassJob::Run()
182{
183    panda_file::File::EntityId recordId(classId_);
184    auto *program = new panda::pandasm::Program();
185    std::string record_name = "";
186    compiler_.CompileAbcClass(recordId, *program, record_name);
187
188    // Update version for abc input when needed
189    if (options_.updatePkgVersionForAbcInput && pkgVersionUpdateRequiredInAbc_) {
190        panda::Timer::timerStart(panda::EVENT_UPDATE_ABC_PKG_VERSION, record_name);
191        UpdatePackageVersion(program, options_);
192        panda::Timer::timerEnd(panda::EVENT_UPDATE_ABC_PKG_VERSION, record_name);
193        // Remove redundant strings created due to version replacement
194        panda::Timer::timerStart(panda::EVENT_UPDATE_ABC_PROGRAM_STRING, record_name);
195        if (options_.removeRedundantFile && hasUpdatedVersion_) {
196            program->strings.clear();
197            for (const auto &[_, function] : program->function_table) {
198                const auto &funcStringSet = function.CollectStringsFromFunctionInsns();
199                program->strings.insert(funcStringSet.begin(), funcStringSet.end());
200            }
201        }
202        panda::Timer::timerEnd(panda::EVENT_UPDATE_ABC_PROGRAM_STRING, record_name);
203    }
204
205    panda::Timer::timerStart(panda::EVENT_UPDATE_ABC_PROG_CACHE, record_name);
206    {
207        std::unique_lock<std::mutex> lock(CompileFileJob::globalMutex_);
208        ASSERT(compiler_.GetAbcFile().GetFilename().find(util::CHAR_VERTICAL_LINE) == std::string::npos);
209        ASSERT(program->record_table.size() == 1);
210        ASSERT(util::RecordNotGeneratedFromBytecode(program->record_table.begin()->first));
211        auto name = compiler_.GetAbcFile().GetFilename();
212        name += util::CHAR_VERTICAL_LINE + program->record_table.begin()->first;
213        auto *cache = allocator_->New<util::ProgramCache>(std::move(*program));
214        progsInfo_.emplace(name, cache);
215    }
216    panda::Timer::timerEnd(panda::EVENT_UPDATE_ABC_PROG_CACHE, record_name);
217
218    delete program;
219    program = nullptr;
220}
221
222void CompileAbcClassJob::UpdateDynamicImportPackageVersion(panda::pandasm::Program *prog,
223    const std::unordered_map<std::string, panda::es2panda::PkgInfo> &pkgContextInfo)
224{
225    for (auto &[name, function] : prog->function_table) {
226        util::VisitDyanmicImports<false>(function, [this, &prog, pkgContextInfo](std::string &ohmurl) {
227            const auto &newOhmurl = util::UpdatePackageVersionIfNeeded(ohmurl, pkgContextInfo);
228            if (newOhmurl == ohmurl) {
229                return;
230            }
231            this->SetHasUpdatedVersion(true);
232            prog->strings.insert(newOhmurl);
233            ohmurl = newOhmurl;
234        });
235    }
236}
237
238void CompileAbcClassJob::UpdateStaticImportPackageVersion(panda::pandasm::Program *prog,
239    const std::unordered_map<std::string, panda::es2panda::PkgInfo> &pkgContextInfo)
240{
241    for (auto &[recordName, record] : prog->record_table) {
242        util::VisitStaticImports<false>(*prog, record, [this, pkgContextInfo](std::string &ohmurl) {
243            const auto &newOhmurl = util::UpdatePackageVersionIfNeeded(ohmurl, pkgContextInfo);
244            if (newOhmurl == ohmurl) {
245                return;
246            }
247            this->SetHasUpdatedVersion(true);
248            ohmurl = newOhmurl;
249        });
250    }
251}
252
253void CompileAbcClassJob::UpdatePackageVersion(panda::pandasm::Program *prog,
254                                              const panda::es2panda::CompilerOptions &options)
255{
256    bool isAccurateUpdateVersion = !options.compileContextInfo.updateVersionInfo.empty();
257    const std::unordered_map<std::string, panda::es2panda::PkgInfo> &pkgContextInfo = isAccurateUpdateVersion ?
258        options.compileContextInfo.updateVersionInfo.at(abcPkgName_) : options.compileContextInfo.pkgContextInfo;
259    // Replace for esm module static import
260    UpdateStaticImportPackageVersion(prog, pkgContextInfo);
261    // Replace for dynamic import
262    UpdateDynamicImportPackageVersion(prog, pkgContextInfo);
263}
264
265void PostAnalysisOptimizeFileJob::Run()
266{
267    util::Helpers::OptimizeProgram(program_, fileName_);
268    panda::Timer::timerEnd(panda::EVENT_OPTIMIZE_PROGRAM, fileName_);
269}
270
271void CompileFuncQueue::Schedule()
272{
273    ASSERT(jobsCount_ == 0);
274    std::unique_lock<std::mutex> lock(m_);
275    const auto &functions = context_->Binder()->Functions();
276
277    for (auto *function : functions) {
278        auto *funcJob = new CompileFunctionJob(context_);
279        funcJob->SetFunctionScope(function);
280        jobs_.push_back(funcJob);
281        jobsCount_++;
282    }
283
284    if (context_->Binder()->Program()->Kind() == parser::ScriptKind::MODULE) {
285        auto *moduleRecordJob = new CompileModuleRecordJob(context_);
286        jobs_.push_back(moduleRecordJob);
287        jobsCount_++;
288    }
289
290    lock.unlock();
291    jobsAvailable_.notify_all();
292}
293
294void CompileFileQueue::Schedule()
295{
296    ASSERT(jobsCount_ == 0);
297    std::unique_lock<std::mutex> lock(m_);
298
299    for (auto &input: options_->sourceFiles) {
300        auto *fileJob = new CompileFileJob(&input, options_, progsInfo_, optimizationPendingProgs_,
301                                           symbolTable_, allocator_);
302        jobs_.push_back(fileJob);
303        jobsCount_++;
304    }
305
306    lock.unlock();
307    jobsAvailable_.notify_all();
308}
309
310bool CompileAbcClassQueue::NeedUpdateVersion()
311{
312    std::unordered_map<std::string, std::unordered_map<std::string, panda::es2panda::PkgInfo>> updateVersionInfo =
313        options_.compileContextInfo.updateVersionInfo;
314    auto iter = updateVersionInfo.find(src_->pkgName);
315    return updateVersionInfo.empty() || (iter != updateVersionInfo.end() && !iter->second.empty());
316}
317
318void CompileAbcClassQueue::Schedule()
319{
320    std::unique_lock<std::mutex> lock(m_);
321
322    auto classIds = compiler_.GetAbcFile().GetClasses();
323    size_t expectedProgsCountInAbcFile = 0;
324    bool needUpdateVersion = NeedUpdateVersion();
325    for (size_t i = 0; i != classIds.size(); ++i) {
326        if (!compiler_.CheckClassId(classIds[i], i)) {
327            continue;
328        }
329
330        auto *abcClassJob = new CompileAbcClassJob(classIds[i], options_, compiler_, progsInfo_, allocator_,
331                                                   src_->pkgName, needUpdateVersion);
332
333        jobs_.push_back(abcClassJob);
334        jobsCount_++;
335        expectedProgsCountInAbcFile++;
336    }
337    {
338        std::unique_lock<std::mutex> lock(globalMutex_);
339        Compiler::SetExpectedProgsCount(Compiler::GetExpectedProgsCount() + expectedProgsCountInAbcFile - 1);
340    }
341
342    lock.unlock();
343    jobsAvailable_.notify_all();
344}
345
346void PostAnalysisOptimizeFileQueue::Schedule()
347{
348    ASSERT(jobsCount_ == 0);
349    std::unique_lock<std::mutex> lock(m_);
350
351    for (const auto &optimizationPendingProgName : optimizationPendingProgs_) {
352        auto progInfo = progsInfo_.find(optimizationPendingProgName);
353        if (progInfo == progsInfo_.end()) {
354            continue;
355        }
356        auto *optimizeJob = new PostAnalysisOptimizeFileJob(progInfo->first, &progInfo->second->program);
357        jobs_.push_back(optimizeJob);
358        jobsCount_++;
359    }
360}
361
362}  // namespace panda::es2panda::compiler
363