1pub const KERNEL_CU_NAMES: &[&str] = &[
12 "join",
13 "dedup",
14 "groupby",
15 "scan",
16 "sort",
17 "filter",
18 "set_ops",
19 "pack",
20 "pir",
21 "cnf",
22 "cache",
23 "weights",
24 "circuit",
25 "mc_sample",
26 "mc_eval",
27 "arith",
28 "sat",
29 "d4",
30 "neural",
31 "ilp",
32 "ilp_credit",
33 "ilp_exact",
34 "ilp_exact_nary",
35 "epistemic",
36 "wcoj",
37 "mc_resident",
38 "joint_solve",
39 "resident_relational",
40 "resident_filter_project",
41 "resident_schedule",
42];
43
44pub struct KernelModuleSpec {
47 pub cu_name: &'static str,
48 pub module_name: &'static str,
49 pub kernels: &'static [&'static str],
50}
51
52pub const KERNEL_MODULES: &[KernelModuleSpec] = &[
55 KernelModuleSpec {
56 cu_name: "join",
57 module_name: "xlog_join",
58 kernels: &[
59 "hash_join_build",
60 "hash_join_probe",
61 "compute_composite_hash",
62 "hash_join_bucket_count_v2",
63 "hash_join_scatter_v2",
64 "hash_join_probe_v2",
65 "hash_join_probe_v2_count_per_row",
66 "hash_join_probe_v2_materialize",
67 "hash_join_total_from_scan",
68 "hash_join_csm_unmatched_mask",
69 "hash_join_semi",
70 "hash_join_anti",
71 "init_hash_table",
72 "nested_loop_join_inner_u32_1key_pairs",
74 "sort_merge_join_inner_u32_1key_pairs",
77 ],
78 },
79 KernelModuleSpec {
80 cu_name: "dedup",
81 module_name: "xlog_dedup",
82 kernels: &[
83 "mark_duplicates",
84 "mark_unique_columnar",
85 "mark_unique_and_scan_columnar",
86 "compact_rows",
87 "mark_unique_full_row_bytewise",
88 "mark_diff_full_row_typed_sorted",
89 "small_sort_full_row_indices_typed",
90 ],
91 },
92 KernelModuleSpec {
93 cu_name: "groupby",
94 module_name: "xlog_groupby",
95 kernels: &[
96 "detect_group_boundaries",
97 "detect_boundaries",
98 "extract_group_keys",
99 "group_ids_from_boundaries",
100 "group_start_indices",
101 "capture_num_groups",
102 "groupby_count",
103 "groupby_sum",
104 "groupby_sum_u64",
105 "groupby_min",
106 "groupby_min_u64",
107 "groupby_max",
108 "groupby_max_u64",
109 "groupby_logsumexp_max",
110 "groupby_logsumexp_sumexp",
111 "groupby_logsumexp_final",
112 ],
113 },
114 KernelModuleSpec {
115 cu_name: "scan",
116 module_name: "xlog_scan",
117 kernels: &[
118 "block_inclusive_scan",
119 "add_block_offsets",
120 "exclusive_scan_mask",
121 "count_mask",
122 "multiblock_scan_phase1",
123 "multiblock_scan_u32_phase1",
124 "multiblock_scan_phase2",
125 "multiblock_scan_phase3",
126 ],
127 },
128 KernelModuleSpec {
129 cu_name: "sort",
130 module_name: "xlog_sort",
131 kernels: &[
132 "radix_histogram",
133 "radix_scatter",
134 "compute_ranks",
135 "radix_scatter_stable",
136 "compute_digit_prefix_sums",
137 "init_indices",
138 "apply_permutation_u32",
139 "apply_permutation_bytes",
140 "gather_keys_i32_ordered_u32",
141 "gather_keys_f32_ordered_u32",
142 "gather_keys_bool_ordered_u32",
143 "gather_keys_u64_lo_u32",
144 "gather_keys_u64_hi_u32",
145 "gather_keys_i64_lo_u32",
146 "gather_keys_i64_hi_u32",
147 "gather_keys_f64_lo_u32",
148 "gather_keys_f64_hi_u32",
149 "check_ascending_sorted_u32",
152 ],
153 },
154 KernelModuleSpec {
155 cu_name: "filter",
156 module_name: "xlog_filter",
157 kernels: &[
158 "filter_compare_u32",
159 "filter_compare_i64",
160 "filter_compare_f64",
161 "filter_compare_i32",
162 "filter_compare_u64",
163 "filter_compare_f32",
164 "filter_compare_u8",
165 "filter_compare_u32_scan_phase1",
166 "filter_compare_f64_scan_phase1",
167 "filter_compare_f32_scan_phase1",
168 "filter_compare_u32_col",
169 "filter_compare_i32_col",
170 "filter_compare_i64_col",
171 "filter_compare_u64_col",
172 "filter_compare_f32_col",
173 "filter_compare_f64_col",
174 "filter_compare_u8_col",
175 "fill_u32_iota",
176 "fill_u32_const",
177 "mark_random_vars",
178 "random_var_to_bit_from_list",
179 "check_random_var_count",
180 "compact_u32_by_mask",
181 "compact_i64_by_mask",
182 "compact_f64_by_mask",
183 "compact_bytes_by_mask",
184 "capture_compact_count",
185 "mask_clamp_rows",
186 "mask_and",
187 "mask_or",
188 "mask_not",
189 ],
190 },
191 KernelModuleSpec {
192 cu_name: "set_ops",
193 module_name: "xlog_set_ops",
194 kernels: &["concat_u32", "concat_bytes", "sorted_diff_mark"],
195 },
196 KernelModuleSpec {
197 cu_name: "pack",
198 module_name: "xlog_pack",
199 kernels: &[
200 "pack_keys",
201 "hash_packed_keys",
202 "pack_and_hash_keys",
203 "pack_and_hash_keys_generic",
204 "pack_keys_aligned",
205 "unpack_column",
206 "unpack_column_counted",
207 "gather_packed_rows",
208 "gather_packed_rows_counted",
209 "scatter_packed_rows",
210 "compare_packed_keys",
211 "pack_bools_to_bitmap",
212 ],
213 },
214 KernelModuleSpec {
215 cu_name: "pir",
216 module_name: "xlog_pir",
217 kernels: &[
218 "pir_pack_keys",
219 "pir_hash_keys",
220 "pir_mark_unique",
221 "pir_find_existing",
222 "pir_mark_new_groups",
223 "pir_build_group_ids",
224 "pir_fill_child_parents",
225 "pir_mark_unique_pairs",
226 "pir_compact_pairs",
227 "pir_count_children",
228 "pir_write_child_offsets",
229 "pir_gather_children",
230 "pir_build_graph_child_counts",
231 "pir_sum_counts",
232 "pir_emit_nodes_and_ids",
233 "pir_update_counts",
234 ],
235 },
236 KernelModuleSpec {
237 cu_name: "cnf",
238 module_name: "xlog_cnf",
239 kernels: &[
240 "cnf_reachability_init",
241 "cnf_reachability_bfs",
242 "cnf_mark_leaf_choice",
243 "cnf_assign_leaf_var",
244 "cnf_assign_choice_var",
245 "cnf_mark_node_vars",
246 "cnf_count_clauses",
247 "cnf_capture_last_counts",
248 "cnf_compute_leaf_choice_totals",
249 "cnf_compute_totals",
250 "cnf_assign_node_var",
251 "cnf_emit_clauses",
252 "cnf_set_clause_end",
253 ],
254 },
255 KernelModuleSpec {
256 cu_name: "cache",
257 module_name: "xlog_cache",
258 kernels: &[
259 "cache_cnf_hash",
260 "cache_lookup_or_insert",
261 "cache_evict_lru",
262 "cache_store_u8",
263 "cache_store_u32",
264 "cache_store_i32",
265 "cache_store_f64",
266 "cache_store_meta",
267 ],
268 },
269 KernelModuleSpec {
270 cu_name: "weights",
271 module_name: "xlog_weights",
272 kernels: &[
273 "weights_fill_leaf",
274 "weights_fill_choice",
275 "weights_count_lift_exact",
276 "weights_set_evidence_from_nodes",
277 "weights_apply_evidence",
278 "weights_map_nodes_to_vars",
279 "weights_force_var_false",
280 "weights_restore_var_false",
281 "weights_force_var_true",
282 "weights_restore_var_true",
283 "weights_copy_slot_to_batch",
284 "weights_apply_query_vars",
285 "weights_restore_query_vars",
286 "weights_apply_query_vars_false_batched",
287 "weights_restore_query_vars_false_batched",
288 "weights_apply_query_vars_true_batched",
289 "weights_restore_query_vars_true_batched",
290 ],
291 },
292 KernelModuleSpec {
293 cu_name: "circuit",
294 module_name: "xlog_circuit",
295 kernels: &[
296 "xgcf_forward_level",
297 "xgcf_backward_level_propagate",
298 "xgcf_backward_level_decision_grad",
299 "xgcf_backward_level_lit_grad",
300 "xgcf_free_var_apply_grad",
301 "xgcf_free_var_reduce_stage",
302 "xgcf_add_scalar",
303 "xgcf_forward_level_cached",
304 "xgcf_eval_all_levels_cached",
305 "xgcf_eval_all_levels_cached_batched",
306 "xgcf_backward_level_propagate_cached",
307 "xgcf_backward_level_decision_grad_cached",
308 "xgcf_backward_level_lit_grad_cached",
309 "xgcf_backward_all_levels_cached",
310 "xgcf_backward_all_levels_cached_batched",
311 "xgcf_free_var_apply_grad_cached",
312 "xgcf_free_var_reduce_stage_cached",
313 "xgcf_add_scalar_cached",
314 "xgcf_set_root_adj_cached_batched",
315 "xgcf_copy_root_cached",
316 "xgcf_copy_root_cached_meta",
317 "xgcf_copy_root_cached_meta_batched",
318 ],
319 },
320 KernelModuleSpec {
321 cu_name: "mc_sample",
322 module_name: "xlog_mc_sample",
323 kernels: &["mc_sample_bernoulli"],
324 },
325 KernelModuleSpec {
326 cu_name: "mc_eval",
327 module_name: "xlog_mc_eval",
328 kernels: &[
329 "mc_eval_mask_var",
330 "mc_eval_mask_ad_choice",
331 "mc_eval_query_evidence_truth",
332 "mc_accumulate_counts",
333 ],
334 },
335 KernelModuleSpec {
336 cu_name: "arith",
337 module_name: "xlog_arith",
338 kernels: &[
339 "arith_binary_i64",
340 "arith_binary_i32",
341 "arith_binary_u64",
342 "arith_binary_u32",
343 "arith_binary_f64",
344 "arith_binary_f32",
345 "arith_abs_i64",
346 "arith_abs_i32",
347 "arith_abs_f64",
348 "arith_abs_f32",
349 "arith_pow_f64",
350 "arith_cast",
351 "arith_fill_const_u32",
352 "arith_fill_const_u64",
353 "arith_fill_const_i64",
354 "arith_fill_const_i32",
355 "arith_fill_const_f64",
356 "arith_fill_const_f32",
357 "arith_fill_const_u8",
358 "arith_select_i64",
359 "arith_select_i32",
360 "arith_select_u64",
361 "arith_select_u32",
362 "arith_select_f64",
363 "arith_select_f32",
364 ],
365 },
366 KernelModuleSpec {
367 cu_name: "sat",
368 module_name: "xlog_sat",
369 kernels: &[
370 "sat_cdcl_solve",
371 "sat_check_model",
372 "sat_proof_mark_needed",
373 "sat_proof_check",
374 "sat_assert_status",
375 "sat_assert_ok",
376 "sat_xgcf_cnf_counts",
377 "sat_xgcf_cnf_emit",
378 "sat_xgcf_cnf_capture_last_counts",
379 "sat_xgcf_cnf_compute_totals",
380 "sat_cnf_write_terminator",
381 "sat_cnf_copy_into",
382 "sat_shift_offsets",
383 "sat_xgcf_write_root_unit_clause",
384 "sat_not_phi_counts",
385 "sat_emit_not_phi",
386 ],
387 },
388 KernelModuleSpec {
389 cu_name: "d4",
390 module_name: "xlog_d4",
391 kernels: &[
392 "d4_validate_cnf",
393 "d4_levelize_counts",
394 "d4_levelize_emit",
395 "d4_frontier_prepare",
396 "d4_frontier_expand",
397 "d4_compile_count",
398 "d4_compile_emit",
399 "d4_capture_emit_meta",
400 "d4_support_level",
401 "d4_support_set_root_bits",
402 "d4_smooth_count",
403 "d4_smooth_wrapper_counts",
404 "d4_smooth_wrapper_edge_counts_or",
405 "d4_smooth_wrapper_edge_counts_dec",
406 "d4_smooth_init_nodes",
407 "d4_smooth_emit_level",
408 "d4_smooth_check_edge_cap",
409 "d4_mark_vars_in_clauses",
410 "d4_mark_vars_in_circuit",
411 "d4_build_free_var_mask",
412 "d4_assert_u32_eq",
413 "d4_assert_bitset_var",
414 "d4_assert_leaf_root_and_degree",
415 ],
416 },
417 KernelModuleSpec {
418 cu_name: "neural",
419 module_name: "xlog_neural",
420 kernels: &[
421 "neural_fill_ad_chain_f32",
422 "neural_scatter_ad_chain_grads_f32",
423 ],
424 },
425 KernelModuleSpec {
426 cu_name: "ilp",
427 module_name: "xlog_ilp",
428 kernels: &[
429 "extract_nonzero_indices",
430 "ilp_mark_selected_ids_u32",
431 "ilp_mark_selected_ids_i32",
432 "ilp_mark_selected_ids_i64",
433 "ilp_mark_selected_ids_u64",
434 "ilp_validate_selected_ids_u32",
435 "ilp_validate_selected_ids_i32",
436 "ilp_validate_selected_ids_i64",
437 "ilp_validate_selected_ids_u64",
438 "ilp_broadcast_candidate_flag",
439 "ilp_coo_fill_from_mask",
440 "ilp_csr_histogram",
441 "ilp_reduce_sum_f32",
442 "ilp_reduce_sum_f64",
443 ],
444 },
445 KernelModuleSpec {
446 cu_name: "ilp_credit",
447 module_name: "xlog_ilp_credit",
448 kernels: &[
449 "ilp_coo_fill",
450 "ilp_credit_forward_f32",
451 "ilp_credit_forward_f64",
452 "ilp_credit_backward_f32",
453 "ilp_credit_backward_f64",
454 ],
455 },
456 KernelModuleSpec {
457 cu_name: "ilp_exact",
458 module_name: "xlog_ilp_exact",
459 kernels: &[
460 "ilp_exact_score",
461 "ilp_exact_score_u32",
462 "ilp_exact_score_chain_smem",
463 "ilp_exact_score_chain_smem_u32",
464 "ilp_exact_select_topk",
465 ],
466 },
467 KernelModuleSpec {
468 cu_name: "ilp_exact_nary",
469 module_name: "xlog_ilp_exact_nary",
470 kernels: &["ilp_exact_nary_score"],
471 },
472 KernelModuleSpec {
473 cu_name: "epistemic",
474 module_name: "xlog_epistemic",
475 kernels: &[
476 "epistemic_generate_candidate_assumptions_u8",
477 "epistemic_propagate_candidates_u8",
478 "epistemic_validate_candidate_bits_u8",
479 "epistemic_populate_model_membership_u8",
480 "epistemic_populate_model_membership_from_tuple_source_u8",
481 "epistemic_populate_model_membership_from_tuple_source_arity1_u8",
482 "epistemic_populate_model_membership_from_tuple_source_arity2_u8",
483 "epistemic_populate_model_membership_from_tuple_source_arity3_u8",
484 "epistemic_populate_model_membership_from_tuple_source_arity_n_u8",
485 "epistemic_validate_world_views_u8",
486 "epistemic_validate_constraints_u8",
487 "epistemic_materialize_accepted_candidates_u8",
488 "epistemic_materialize_final_result_flags_u8",
489 "epistemic_build_final_tuple_row_map_u8",
490 "epistemic_close_final_tuple_rejections_u8",
491 "epistemic_materialize_final_tuple_column_u8",
492 ],
493 },
494 KernelModuleSpec {
495 cu_name: "wcoj",
496 module_name: "xlog_wcoj",
497 kernels: &[
498 "wcoj_build_metadata_mark_boundaries_u32",
499 "wcoj_build_metadata_mark_boundaries_u64",
500 "wcoj_build_metadata_scatter_u32",
501 "wcoj_build_metadata_scatter_u64",
502 "wcoj_triangle_build_hg_work_plan_u32",
503 "wcoj_triangle_count_hg_u32",
504 "wcoj_triangle_groupby_root_count_hg_u32",
505 "wcoj_triangle_groupby_root_sum_hg_u32",
506 "wcoj_triangle_groupby_root_min_hg_u32",
507 "wcoj_triangle_groupby_root_max_hg_u32",
508 "wcoj_triangle_materialize_hg_u32",
509 "wcoj_triangle_build_hg_work_plan_u64",
510 "wcoj_triangle_count_hg_u64",
511 "wcoj_triangle_groupby_root_count_hg_u64",
512 "wcoj_triangle_groupby_root_sum_hg_u64",
513 "wcoj_triangle_groupby_root_min_hg_u64",
514 "wcoj_triangle_groupby_root_max_hg_u64",
515 "wcoj_groupby_root_segment_sum_counts_u32",
516 "wcoj_groupby_root_segment_sum_values_u64",
517 "wcoj_groupby_root_segment_min_values_u64",
518 "wcoj_groupby_root_segment_max_values_u64",
519 "wcoj_triangle_materialize_hg_u64",
520 "wcoj_triangle_count_hg_cached_u32",
521 "wcoj_triangle_materialize_hg_cached_u32",
522 "wcoj_scan_hg_block_counts_u32",
523 "wcoj_compute_total",
524 "wcoj_layout_check_sorted_unique_u32",
525 "wcoj_layout_check_sorted_unique_u64",
526 "wcoj_4cycle_build_e2_work_prefix_u32",
527 "wcoj_4cycle_build_hg_work_plan_u32",
528 "wcoj_4cycle_count_hg_u32",
529 "wcoj_4cycle_groupby_root_count_hg_u32",
530 "wcoj_4cycle_groupby_root_sum_hg_u32",
531 "wcoj_4cycle_groupby_root_min_hg_u32",
532 "wcoj_4cycle_groupby_root_max_hg_u32",
533 "wcoj_4cycle_materialize_hg_u32",
534 "wcoj_4cycle_build_e2_work_prefix_u64",
535 "wcoj_4cycle_build_hg_work_plan_u64",
536 "wcoj_4cycle_count_hg_u64",
537 "wcoj_4cycle_groupby_root_count_hg_u64",
538 "wcoj_4cycle_materialize_hg_u64",
539 "wcoj_clique5_count_hg_u32",
543 "wcoj_clique5_materialize_hg_u32",
544 "wcoj_clique5_count_hg_u64",
545 "wcoj_clique5_materialize_hg_u64",
546 "wcoj_clique6_count_hg_u32",
547 "wcoj_clique6_materialize_hg_u32",
548 "wcoj_clique6_count_hg_u64",
549 "wcoj_clique6_materialize_hg_u64",
550 "wcoj_clique7_count_hg_u32",
551 "wcoj_clique7_materialize_hg_u32",
552 "wcoj_clique7_count_hg_u64",
553 "wcoj_clique7_materialize_hg_u64",
554 "wcoj_clique8_count_hg_u32",
555 "wcoj_clique8_materialize_hg_u32",
556 "wcoj_clique8_count_hg_u64",
557 "wcoj_clique8_materialize_hg_u64",
558 "wcoj_clique5_groupby_root_count_hg_u32",
561 "wcoj_clique6_groupby_root_count_hg_u32",
562 "fj_expand_work_prefix_u32",
564 "fj_expand_count_u32",
565 "fj_expand_emit_u32",
566 "fj_probe_refine_u32",
567 "fj_expand_count_u64",
570 "fj_expand_emit_u64",
571 "fj_probe_refine_u64",
572 "fj_count_multiplicity",
574 "fj_delta_range_u32",
577 "fj_delta_mark_u32",
578 "fj_delta_subtract_u32",
579 "fj_delta_popcount",
580 "fj_delta_emit_u32",
581 "fj_delta_max_u32",
582 "fj_delta_sparse_estimate",
584 "fj_delta_sparse_load_r",
585 "fj_delta_sparse_insert_candidates",
586 "fj_delta_sparse_mark",
587 "fj_delta_sparse_emit",
588 ],
589 },
590 KernelModuleSpec {
591 cu_name: "mc_resident",
592 module_name: "xlog_mc_resident",
593 kernels: &["mc_resident_engine"],
594 },
595 KernelModuleSpec {
596 cu_name: "joint_solve",
597 module_name: "xlog_joint_solve",
598 kernels: &[
599 "joint_label_feasibility",
600 "joint_label_top2",
601 "joint_component_plan_init",
602 "joint_component_entity_owners",
603 "joint_component_union",
604 "joint_component_compress",
605 "joint_component_enumerate",
606 "joint_component_chain_dp",
607 "joint_component_branch_and_bound",
608 ],
609 },
610 KernelModuleSpec {
611 cu_name: "resident_relational",
612 module_name: "xlog_resident_relational",
613 kernels: &[
614 "resident_set_insert",
615 "resident_set_finalize",
616 "resident_join_build",
617 "resident_join_probe_inner",
618 "resident_join_probe_semi",
619 "resident_join_finalize",
620 "resident_control_initialize",
621 "resident_scc_begin",
622 "resident_changed_reset",
623 "resident_changed_mark",
624 "resident_convergence",
625 "resident_terminal_success",
626 "resident_test_status",
627 "resident_trace_initialize",
628 "resident_trace_increment",
629 "resident_schema_winners_initialize",
630 "resident_schema_winner_mark",
631 "resident_receipt_pack",
632 ],
633 },
634 KernelModuleSpec {
635 cu_name: "resident_filter_project",
636 module_name: "xlog_resident_filter_project",
637 kernels: &[
638 "resident_filter_mask_scan",
639 "resident_filter_scan_blocks",
640 "resident_filter_add_offsets",
641 "resident_filter_finalize",
642 "resident_filter_compact",
643 "resident_project_finalize",
644 "resident_project_materialize",
645 ],
646 },
647 KernelModuleSpec {
648 cu_name: "resident_schedule",
649 module_name: "xlog_resident_schedule",
650 kernels: &["resident_schedule_execute"],
651 },
652];
653
654#[cfg(test)]
655mod tests {
656 use super::*;
657
658 #[test]
659 fn kernel_modules_matches_cu_names() {
660 assert_eq!(
661 KERNEL_MODULES.len(),
662 KERNEL_CU_NAMES.len(),
663 "KERNEL_MODULES length ({}) != KERNEL_CU_NAMES length ({})",
664 KERNEL_MODULES.len(),
665 KERNEL_CU_NAMES.len(),
666 );
667 for (i, spec) in KERNEL_MODULES.iter().enumerate() {
668 assert_eq!(
669 spec.cu_name, KERNEL_CU_NAMES[i],
670 "KERNEL_MODULES[{}].cu_name = {:?}, expected {:?}",
671 i, spec.cu_name, KERNEL_CU_NAMES[i],
672 );
673 }
674 }
675
676 #[test]
677 fn kernel_modules_count_is_30() {
678 assert_eq!(KERNEL_MODULES.len(), 30);
679 }
680
681 #[test]
682 fn all_kernel_entries_are_non_empty() {
683 for spec in KERNEL_MODULES {
684 assert!(
685 !spec.kernels.is_empty(),
686 "module {:?} has no kernel entries",
687 spec.cu_name,
688 );
689 assert!(
690 !spec.module_name.is_empty(),
691 "module {:?} has empty module_name",
692 spec.cu_name,
693 );
694 }
695 }
696}