NoisePage源码分析（未完待续）

1
llvm::cl::OptionCategory TPL_OPTIONS_CATEGORY("TPL Compiler Options", "Options for controlling the TPL compilation process.");  // NOLINT
2
llvm::cl::opt<bool> PRINT_AST("print-ast", llvm::cl::desc("Print the programs AST"), llvm::cl::cat(TPL_OPTIONS_CATEGORY));  // NOLINT
3
llvm::cl::opt<bool> PRINT_TBC("print-tbc", llvm::cl::desc("Print the generated TPL Bytecode"), llvm::cl::cat(TPL_OPTIONS_CATEGORY));  // NOLINT
4
llvm::cl::opt<bool> PRETTY_PRINT("pretty-print", llvm::cl::desc("Pretty-print the source from the parsed AST"), llvm::cl::cat(TPL_OPTIONS_CATEGORY));  // NOLINT
5
llvm::cl::opt<bool> IS_SQL("sql", llvm::cl::desc("Is the input a SQL query?"), llvm::cl::cat(TPL_OPTIONS_CATEGORY));  // NOLINT
6
llvm::cl::opt<bool> TPCH("tpch", llvm::cl::desc("Should the TPCH database be loaded? Requires '-schema' and '-data' directories."), llvm::cl::cat(TPL_OPTIONS_CATEGORY));  // NOLINT
7
llvm::cl::opt<std::string> DATA_DIR("data", llvm::cl::desc("Where to find data files of tables to load"), llvm::cl::cat(TPL_OPTIONS_CATEGORY));  // NOLINT
8
llvm::cl::opt<std::string> INPUT_FILE(llvm::cl::Positional, llvm::cl::desc("<input file>"), llvm::cl::init(""), llvm::cl::cat(TPL_OPTIONS_CATEGORY));  // NOLINT
9
llvm::cl::opt<std::string> OUTPUT_NAME("output-name", llvm::cl::desc("Print the output name"), llvm::cl::init("schema10"), llvm::cl::cat(TPL_OPTIONS_CATEGORY));  // NOLINT
10
llvm::cl::opt<std::string> HANDLERS_PATH("handlers-path", llvm::cl::desc("Path to the bytecode handlers bitcode file"), llvm::cl::init("./bytecode_handlers_ir.bc"), llvm::cl::cat(TPL_OPTIONS_CATEGORY));  // NOLINT

1
// This executable reads the unoptimized bitcode file and:
2
// 1. Converts to an LLVM Module
3
// 2. Removes the static global variable
4
// 3. Modifies linkage types of all defined functions to LinkOnce
5
// 4. Cleans up function arguments
6
// 5. Writes out optimized module as bitcode file

1
void RemoveGlobalUses(llvm::Module *module) {
2
  // When we created the original bitcode file, we forced all functions to be
3
  // generated by storing their address in a global variable. We delete this
4
  // variable now so the final binary can be made smaller by eliminating unused
5
  // ops.
6
  auto var = module->getGlobalVariable(GLOBAL_VAR_NAME);
7
  if (var != nullptr) {
8
    var->replaceAllUsesWith(llvm::UndefValue::get(var->getType()));
9
    var->eraseFromParent();
10
  }
11

12
  // Clang created a global variable holding all force-used items. Delete it.
13
  auto used = module->getGlobalVariable(LLVM_COMPILED_USED);
14
  if (used != nullptr) {
15
    used->eraseFromParent();
16
  }
17
}

1
TEST_F(BinderCorrectnessTest, SelectStatementComplexTest) {
2
  // Test regular table name
3
  BINDER_LOG_DEBUG("Parsing sql query");
4
  std::string select_sql =
5
      "SELECT A.A1, B.B2 FROM A INNER JOIN b ON a.a1 = b.b1 WHERE a1 < 100 "
6
      "GROUP BY A.a1, B.b2 HAVING a1 > 50 ORDER BY a1";
7

8
  auto parse_tree = parser::PostgresParser::BuildParseTree(select_sql);
9
  auto statement = parse_tree->GetStatements()[0];
10
  binder_->BindNameToNode(common::ManagedPointer(parse_tree), nullptr, nullptr);
11
  auto select_stmt = statement.CastManagedPointerTo<parser::SelectStatement>();
12
  EXPECT_EQ(0, select_stmt->GetDepth());
13

14
  // Check select_list
15
  BINDER_LOG_DEBUG("Checking select list");
16
  auto col_expr = select_stmt->GetSelectColumns()[0].CastManagedPointerTo<parser::ColumnValueExpression>();
17
  EXPECT_EQ(col_expr->GetDatabaseOid(), db_oid_);              // A.a1
18
  EXPECT_EQ(col_expr->GetTableOid(), table_a_oid_);            // A.a1
19
  EXPECT_EQ(col_expr->GetColumnOid(), catalog::col_oid_t(1));  // A.a1; columns are indexed from 1
20
  EXPECT_EQ(execution::sql::SqlTypeId::Integer, col_expr->GetReturnValueType());
21
  EXPECT_EQ(0, col_expr->GetDepth());

1
namespace noisepage {
2
void binder::SqlNodeVisitor::Visit(common::ManagedPointer<parser::AggregateExpression> expr) {
3
  expr->AcceptChildren(common::ManagedPointer(this));
4
}
5
void binder::SqlNodeVisitor::Visit(common::ManagedPointer<parser::CaseExpression> expr) {
6
  expr->AcceptChildren(common::ManagedPointer(this));
7
}
8
void binder::SqlNodeVisitor::Visit(common::ManagedPointer<parser::ColumnValueExpression> expr) {
9
  expr->AcceptChildren(common::ManagedPointer(this));
10
}
11
void binder::SqlNodeVisitor::Visit(common::ManagedPointer<parser::ComparisonExpression> expr) {
12
  expr->AcceptChildren(common::ManagedPointer(this));
13
}
14
void binder::SqlNodeVisitor::Visit(common::ManagedPointer<parser::ConjunctionExpression> expr) {
15
  expr->AcceptChildren(common::ManagedPointer(this));
16
}
17
void binder::SqlNodeVisitor::Visit(common::ManagedPointer<parser::ConstantValueExpression> expr) {
18
  expr->AcceptChildren(common::ManagedPointer(this));
19
}
20
void binder::SqlNodeVisitor::Visit(common::ManagedPointer<parser::DefaultValueExpression> expr) {
21
  expr->AcceptChildren(common::ManagedPointer(this));
22
}
23
void binder::SqlNodeVisitor::Visit(common::ManagedPointer<parser::DerivedValueExpression> expr) {
24
  expr->AcceptChildren(common::ManagedPointer(this));
25
}
26
void binder::SqlNodeVisitor::Visit(common::ManagedPointer<parser::FunctionExpression> expr) {
27
  expr->AcceptChildren(common::ManagedPointer(this));
28
}

1
void BytecodeEmitter::EmitDeref(Bytecode bytecode, LocalVar dest, LocalVar src) {
2
  NOISEPAGE_ASSERT(bytecode == Bytecode::Deref1 || bytecode == Bytecode::Deref2 || bytecode == Bytecode::Deref4 ||
3
                       bytecode == Bytecode::Deref8,
4
                   "Bytecode is not a Deref code");
5
  EmitAll(bytecode, dest, src);
6
}
7

8
void BytecodeEmitter::EmitDerefN(LocalVar dest, LocalVar src, uint32_t len) {
9
  EmitAll(Bytecode::DerefN, dest, src, len);
10
}
11

12
void BytecodeEmitter::EmitAssign(Bytecode bytecode, LocalVar dest, LocalVar src) {
13
  NOISEPAGE_ASSERT(bytecode == Bytecode::Assign1 || bytecode == Bytecode::Assign2 || bytecode == Bytecode::Assign4 ||
14
                       bytecode == Bytecode::Assign8,
15
                   "Bytecode is not an Assign code");
16
  EmitAll(bytecode, dest, src);
17
}
18

19
void BytecodeEmitter::EmitAssignImm1(LocalVar dest, int8_t val) { EmitAll(Bytecode::AssignImm1, dest, val); }
20

21
void BytecodeEmitter::EmitAssignImm2(LocalVar dest, int16_t val) { EmitAll(Bytecode::AssignImm2, dest, val); }
22

23
void BytecodeEmitter::EmitAssignImm4(LocalVar dest, int32_t val) { EmitAll(Bytecode::AssignImm4, dest, val); }
24

25
void BytecodeEmitter::EmitAssignImm8(LocalVar dest, int64_t val) { EmitAll(Bytecode::AssignImm8, dest, val); }
26

27
void BytecodeEmitter::EmitAssignImm4F(LocalVar dest, float val) { EmitAll(Bytecode::AssignImm4F, dest, val); }
28

29
void BytecodeEmitter::EmitAssignImm8F(LocalVar dest, double val) { EmitAll(Bytecode::AssignImm8F, dest, val); }
30

31
void BytecodeEmitter::EmitUnaryOp(Bytecode bytecode, LocalVar dest, LocalVar input) { EmitAll(bytecode, dest, input); }
32

33
void BytecodeEmitter::EmitBinaryOp(Bytecode bytecode, LocalVar dest, LocalVar lhs, LocalVar rhs) {
34
  EmitAll(bytecode, dest, lhs, rhs);
35
}

1
class LLVMEngine::TypeMap {
2
 public:
3
  explicit TypeMap(llvm::Module *module) : module_(module) {
4
    llvm::LLVMContext &ctx = module->getContext();
5
    type_map_["nil"] = llvm::Type::getVoidTy(ctx);
6
    type_map_["bool"] = llvm::Type::getInt8Ty(ctx);
7
    type_map_["int8"] = llvm::Type::getInt8Ty(ctx);
8
    type_map_["int16"] = llvm::Type::getInt16Ty(ctx);
9
    type_map_["int32"] = llvm::Type::getInt32Ty(ctx);
10
    type_map_["int64"] = llvm::Type::getInt64Ty(ctx);
11
    type_map_["int128"] = llvm::Type::getInt128Ty(ctx);
12
    type_map_["uint8"] = llvm::Type::getInt8Ty(ctx);
13
    type_map_["uint16"] = llvm::Type::getInt16Ty(ctx);
14
    type_map_["uint32"] = llvm::Type::getInt32Ty(ctx);
15
    type_map_["uint64"] = llvm::Type::getInt64Ty(ctx);
16
    type_map_["uint128"] = llvm::Type::getInt128Ty(ctx);
17
    type_map_["float32"] = llvm::Type::getFloatTy(ctx);
18
    type_map_["float64"] = llvm::Type::getDoubleTy(ctx);
19
    type_map_["string"] = llvm::Type::getInt8PtrTy(ctx);
20
  }
21

22
  /** No copying or moving this class. */
23
  DISALLOW_COPY_AND_MOVE(TypeMap);
24

25
  llvm::Type *VoidType() { return type_map_["nil"]; }
26
  llvm::Type *BoolType() { return type_map_["bool"]; }
27
  llvm::Type *Int8Type() { return type_map_["int8"]; }
28
  llvm::Type *Int16Type() { return type_map_["int16"]; }
29
  llvm::Type *Int32Type() { return type_map_["int32"]; }
30
  llvm::Type *Int64Type() { return type_map_["int64"]; }
31
  llvm::Type *UInt8Type() { return type_map_["uint8"]; }
32
  llvm::Type *UInt16Type() { return type_map_["uint16"]; }
33
  llvm::Type *UInt32Type() { return type_map_["uint32"]; }
34
  llvm::Type *UInt64Type() { return type_map_["uint64"]; }
35
  llvm::Type *Float32Type() { return type_map_["float32"]; }
36
  llvm::Type *Float64Type() { return type_map_["float64"]; }
37
  llvm::Type *StringType() { return type_map_["string"]; }
38

39
  llvm::Type *GetLLVMType(const ast::Type *type);
40

41
 private:
42
  // Given a non-primitive builtin type, convert it to an LLVM type
43
  llvm::Type *GetLLVMTypeForBuiltin(const ast::BuiltinType *builtin_type);
44

45
  // Given a struct type, convert it into an equivalent LLVM struct type
46
  llvm::StructType *GetLLVMStructType(const ast::StructType *struct_type);
47

48
  // Given a TPL function type, convert it into an equivalent LLVM function type
49
  llvm::FunctionType *GetLLVMFunctionType(const ast::FunctionType *func_type);
50

51
 private:
52
  llvm::Module *module_;
53
  std::unordered_map<std::string, llvm::Type *> type_map_;
54
};

1
llvm::StructType *LLVMEngine::TypeMap::GetLLVMStructType(const ast::StructType *struct_type) {
2
  // Collect the fields here
3
  llvm::SmallVector<llvm::Type *, 8> fields;
4

5
  for (const auto &field : struct_type->GetAllFields()) {
6
    fields.push_back(GetLLVMType(field.type_));
7
  }
8

9
  return llvm::StructType::create(fields);
10
}

1
LVMEngine::CompiledModuleBuilder::CompiledModuleBuilder(const CompilerOptions &options,
2
                                                         const BytecodeModule &tpl_module)
3
    : options_(options),
4
      tpl_module_(tpl_module),
5
      target_machine_(nullptr),
6
      context_(std::make_unique<llvm::LLVMContext>()),
7
      llvm_module_(nullptr),
8
      type_map_(nullptr) {
9
  //
10
  // We need to create a suitable TargetMachine for LLVM to before we can JIT
11
  // TPL programs. At the moment, we rely on LLVM to discover all CPU features
12
  // e.g., AVX2 or AVX512, and we make no assumptions about symbol relocations.
13
  //
14
  // TODO(pmenon): This may change with LLVM8 that comes with
15
  // TargetMachineBuilders
16
  // TODO(pmenon): Alter the flags as need be
17
  //
18

19
  const std::string target_triple = llvm::sys::getProcessTriple();
20

21
  {
22
    std::string error;
23
    auto *target = llvm::TargetRegistry::lookupTarget(target_triple, error);
24
    if (target == nullptr) {
25
      EXECUTION_LOG_ERROR("LLVM: Unable to find target with target_triple {}", target_triple);
26
      return;
27
    }
28

29
    // Collect CPU features
30
    llvm::StringMap<bool> feature_map;
31
    if (bool success = llvm::sys::getHostCPUFeatures(feature_map); !success) {
32
      EXECUTION_LOG_ERROR("LLVM: Unable to find all CPU features");
33
      return;
34
    }
35

36
    llvm::SubtargetFeatures target_features;
37
    for (const auto &entry : feature_map) {
38
      target_features.AddFeature(entry.getKey(), entry.getValue());
39
    }
40

41
    EXECUTION_LOG_TRACE("LLVM: Discovered CPU features: {}", target_features.getString());
42

43
    // Both relocation=PIC or JIT=true work. Use the latter for now.
44
    llvm::TargetOptions target_options;
45
    llvm::Optional<llvm::Reloc::Model> reloc;
46
    const llvm::CodeGenOpt::Level opt_level = llvm::CodeGenOpt::Aggressive;
47
    target_machine_.reset(target->createTargetMachine(target_triple, llvm::sys::getHostCPUName(),
48
                                                      target_features.getString(), target_options, reloc, {}, opt_level,
49
                                                      true));
50
    NOISEPAGE_ASSERT(target_machine_ != nullptr, "LLVM: Unable to find a suitable target machine!");
51
  }
52

53
  //
54
  // We've built a TargetMachine we use to generate machine code. Now, we
55
  // load the pre-compiled bytecode module containing all the TPL bytecode
56
  // logic. We add the functions we're about to compile into this module. This
57
  // module forms the unit of JIT.
58
  //
59

60
  {
61
    auto memory_buffer = llvm::MemoryBuffer::getFile(GetEngineSettings()->GetBytecodeHandlersBcPath());
62
    if (auto error = memory_buffer.getError()) {
63
      EXECUTION_LOG_ERROR("There was an error loading the handler bytecode: {}", error.message());
64
    }
65

66
    auto module = llvm::parseBitcodeFile(*(memory_buffer.get()), *context_);
67
    if (!module) {
68
      auto error = llvm::toString(module.takeError());
69
      EXECUTION_LOG_ERROR("{}", error);
70
      throw std::runtime_error(error);
71
    }
72

73
    llvm_module_ = std::move(module.get());
74
    llvm_module_->setModuleIdentifier(tpl_module.GetName());
75
    llvm_module_->setSourceFileName(tpl_module.GetName() + ".tpl");
76
    llvm_module_->setDataLayout(target_machine_->createDataLayout());
77
    llvm_module_->setTargetTriple(target_triple);
78
  }
79

80
  type_map_ = std::make_unique<TypeMap>(llvm_module_.get());
81
}

1
void LLVMEngine::CompiledModuleBuilder::BuildSimpleCFG(const FunctionInfo &func_info,
2
                                                       std::map<std::size_t, llvm::BasicBlock *> *blocks) {
3
  // Before we can generate LLVM IR, we need to build a control-flow graph (CFG) for the function.
4
  // We do this construction directly from the TPL bytecode using a vanilla DFS and produce an
5
  // ordered map ('blocks') from bytecode position to an LLVM basic block. Each entry in the map
6
  // indicates the start of a basic block.
7

8
  // We use this vector as a stack for DFS traversal
9
  llvm::SmallVector<std::size_t, 16> bb_begin_positions = {0};
10

11
  for (auto iter = tpl_module_.GetBytecodeForFunction(func_info); !bb_begin_positions.empty();) {
12
    std::size_t begin_pos = bb_begin_positions.back();
13
    bb_begin_positions.pop_back();
14

15
    // We're at what we think is the start of a new basic block. Scan it until we find a terminal
16
    // instruction. Once we do,
17
    for (iter.SetPosition(begin_pos); !iter.Done(); iter.Advance()) {
18
      Bytecode bytecode = iter.CurrentBytecode();
19

20
      // If the bytecode isn't a terminal for the block, continue until we reach one
21
      if (!Bytecodes::IsTerminal(bytecode)) {
22
        continue;
23
      }
24

25
      // Return?
26
      if (Bytecodes::IsReturn(bytecode)) {
27
        break;
28
      }
29

30
      // Unconditional branch?
31
      if (Bytecodes::IsUnconditionalJump(bytecode)) {
32
        std::size_t branch_target_pos =
33
            iter.GetPosition() + Bytecodes::GetNthOperandOffset(bytecode, 0) + iter.GetJumpOffsetOperand(0);
34

35
        if (blocks->find(branch_target_pos) == blocks->end()) {
36
          (*blocks)[branch_target_pos] = nullptr;
37
          bb_begin_positions.push_back(branch_target_pos);
38
        }
39

40
        break;
41
      }
42

43
      // Conditional branch?
44
      if (Bytecodes::IsConditionalJump(bytecode)) {
45
        std::size_t fallthrough_pos = iter.GetPosition() + iter.CurrentBytecodeSize();
46

47
        if (blocks->find(fallthrough_pos) == blocks->end()) {
48
          bb_begin_positions.push_back(fallthrough_pos);
49
          (*blocks)[fallthrough_pos] = nullptr;
50
        }
51

52
        std::size_t branch_target_pos =
53
            iter.GetPosition() + Bytecodes::GetNthOperandOffset(bytecode, 1) + iter.GetJumpOffsetOperand(1);
54

55
        if (blocks->find(branch_target_pos) == blocks->end()) {
56
          bb_begin_positions.push_back(branch_target_pos);
57
          (*blocks)[branch_target_pos] = nullptr;
58
        }
59

60
        break;
61
      }
62
    }
63
  }
64
}

1
void LLVMEngine::CompiledModuleBuilder::Optimize() {
2
  llvm::legacy::FunctionPassManager function_passes(llvm_module_.get());
3

4
  // Add the appropriate TargetTransformInfo.
5
  function_passes.add(llvm::createTargetTransformInfoWrapperPass(target_machine_->getTargetIRAnalysis()));
6

7
  // Build up optimization pipeline.
8
  llvm::PassManagerBuilder pm_builder;
9
  uint32_t opt_level = 3;
10
  uint32_t size_opt_level = 0;
11
  bool disable_inline_hot_call_site = false;
12
  pm_builder.OptLevel = opt_level;
13
  pm_builder.Inliner = llvm::createFunctionInliningPass(opt_level, size_opt_level, disable_inline_hot_call_site);
14
  pm_builder.populateFunctionPassManager(function_passes);
15

16
  // Add custom passes. Hand-selected based on empirical evaluation.
17
  function_passes.add(llvm::createInstructionCombiningPass());
18
  function_passes.add(llvm::createReassociatePass());
19
  function_passes.add(llvm::createGVNPass());
20
  function_passes.add(llvm::createCFGSimplificationPass());
21
  function_passes.add(llvm::createAggressiveDCEPass());
22
  function_passes.add(llvm::createCFGSimplificationPass());
23

24
  // Run optimization passes on all functions.
25
  function_passes.doInitialization();
26
  for (llvm::Function &func : *llvm_module_) {
27
    function_passes.run(func);
28
  }
29
  function_passes.doFinalization();
30
}

1
void CpuInfo::InitCacheInfo() {
2
#ifdef __APPLE__
3
  // Lookup cache sizes.
4
  std::size_t len = 0;
5
  sysctlbyname("hw.cachesize", nullptr, &len, nullptr, 0);
6
  auto data = std::make_unique<uint64_t[]>(len);
7
  sysctlbyname("hw.cachesize", data.get(), &len, nullptr, 0);
8
  NOISEPAGE_ASSERT(len / sizeof(uint64_t) >= 3, "Expected three levels of cache!");
9

10
  // Copy data
11
  for (uint32_t idx = 0; idx < K_NUM_CACHE_LEVELS; idx++) {
12
    cache_sizes_[idx] = data[idx];
13
  }
14

15
  // Lookup cache line sizes.
16
  std::size_t linesize;
17
  std::size_t sizeof_linesize = sizeof(linesize);
18
  sysctlbyname("hw.cachelinesize", &linesize, &sizeof_linesize, nullptr, 0);
19
  for (auto &cache_line_size : cache_line_sizes_) {
20
    cache_line_size = linesize;
21
  }
22
#else
23
  // Use sysconf to determine cache sizes.
24
  cache_sizes_[L1_CACHE] = static_cast<uint32_t>(sysconf(_SC_LEVEL1_DCACHE_SIZE));
25
  cache_sizes_[L2_CACHE] = static_cast<uint32_t>(sysconf(_SC_LEVEL2_CACHE_SIZE));
26
  cache_sizes_[L3_CACHE] = static_cast<uint32_t>(sysconf(_SC_LEVEL3_CACHE_SIZE));
27

28
  cache_line_sizes_[L1_CACHE] = static_cast<uint32_t>(sysconf(_SC_LEVEL1_DCACHE_LINESIZE));
29
  cache_line_sizes_[L2_CACHE] = static_cast<uint32_t>(sysconf(_SC_LEVEL2_CACHE_LINESIZE));
30
  cache_line_sizes_[L3_CACHE] = static_cast<uint32_t>(sysconf(_SC_LEVEL3_CACHE_LINESIZE));
31
#endif
32
}

1
// #include "execution/util/csv_reader.h" Fix later.

1
int32_t File::Write(const std::byte *data, std::size_t len) const {
2
  NOISEPAGE_ASSERT(IsOpen(), "File must be open before reading");
3
  return HANDLE_EINTR(write(fd_, data, len));
4
}
5

6
int64_t File::Seek(File::Whence whence, int64_t offset) const {
7
  static_assert(sizeof(int64_t) == sizeof(off_t), "off_t must be 64 bits");
8
  return lseek(fd_, static_cast<off_t>(offset), static_cast<int32_t>(whence));
9
}
10

11
bool File::Flush() const {
12
#if defined(OS_LINUX)
13
  return HANDLE_EINTR(fdatasync(fd_)) == 0;
14
#else
15
  return HANDLE_EINTR(fsync(fd_)) == 0;
16
#endif
17
}

1
uint32_t VectorUtil::BitVectorToSelectionVectorDenseAvX2(const uint64_t *bit_vector, uint32_t num_bits,
2
                                                         sel_t *sel_vector) {
3
  // Vector of '8's = [8,8,8,8,8,8,8]
4
  const auto eight = _mm_set1_epi16(8);
5

6
  // Selection vector write index
7
  auto idx = _mm_set1_epi16(0);
8

9
  // Selection vector size
10
  uint32_t k = 0;
11

12
  const uint32_t num_words = common::MathUtil::DivRoundUp(num_bits, 64);
13
  for (uint32_t i = 0; i < num_words; i++) {
14
    uint64_t word = bit_vector[i];
15
    for (uint32_t j = 0; j < 8; j++) {
16
      const auto mask = static_cast<uint8_t>(word);
17
      word >>= 8u;
18
      const __m128i match_pos_scaled =
19
          _mm_loadl_epi64(reinterpret_cast<const __m128i *>(&simd::K8_BIT_MATCH_LUT[mask]));
20
      const __m128i match_pos = _mm_cvtepi8_epi16(match_pos_scaled);
21
      const __m128i pos_vec = _mm_add_epi16(idx, match_pos);
22
      idx = _mm_add_epi16(idx, eight);
23
      _mm_storeu_si128(reinterpret_cast<__m128i *>(sel_vector + k), pos_vec);
24
      k += BitUtil::CountPopulation(static_cast<uint32_t>(mask));
25
    }
26
  }
27

28
  return k;
29
}

1
void *MemoryPool::AllocateAligned(const std::size_t size, const std::size_t alignment, const bool clear) {
2
  void *buf = nullptr;
3

4
  if (size >= mmap_threshold.load(std::memory_order_relaxed)) {
5
    buf = util::Memory::MallocHuge(size, true);
6
    NOISEPAGE_ASSERT(buf != nullptr, "Null memory pointer");
7
    // No need to clear memory, guaranteed on Linux
8
  } else {
9
    if (alignment < MIN_MALLOC_ALIGNMENT) {
10
      if (clear) {
11
        buf = std::calloc(size, 1);
12
      } else {
13
        buf = std::malloc(size);
14
      }
15
    } else {
16
      buf = util::Memory::MallocAligned(size, alignment);
17
      if (clear) {
18
        std::memset(buf, 0, size);
19
      }
20
    }
21
  }