switch root namespace name to NITLOAD

This commit is contained in:
Youmu Liu
2026-09-02 18:15:03 +08:00
parent 4218baad48
commit 7c7101ee48
502 changed files with 710 additions and 721 deletions
+747
View File
@@ -0,0 +1,747 @@
# Format Style Options - Created with Clang Power Tools
---
AccessModifierOffset: -4
AlignAfterOpenBracket: Align
AlignConsecutiveAssignments: false
AlignConsecutiveDeclarations: false
AlignEscapedNewlines: Left
AlignOperands: true
AlignTrailingComments: false
AllowAllParametersOfDeclarationOnNextLine: false
AllowShortBlocksOnASingleLine: false
AllowShortCaseLabelsOnASingleLine: false
AllowShortFunctionsOnASingleLine: None
AllowShortIfStatementsOnASingleLine: false
AllowShortLoopsOnASingleLine: false
AlwaysBreakAfterDefinitionReturnType: None
AlwaysBreakAfterReturnType: None
AlwaysBreakBeforeMultilineStrings: false
AlwaysBreakTemplateDeclarations: false
BinPackArguments: true
BinPackParameters: true
BraceWrapping:
AfterCaseLabel: false
AfterClass: false
AfterControlStatement: false
AfterEnum: false
AfterFunction: true
AfterNamespace: true
AfterObjCDeclaration: false
AfterStruct: false
AfterUnion: false
AfterExternBlock: false
BeforeCatch: false
BeforeElse: false
IndentBraces: false
SplitEmptyFunction: true
SplitEmptyRecord: true
SplitEmptyNamespace: true
BeforeLambdaBody: false
BeforeWhile: false
BreakBeforeBinaryOperators: None
BreakBeforeBraces: Custom
BreakBeforeInheritanceComma: false
BreakBeforeTernaryOperators: false
BreakConstructorInitializers: BeforeComma
BreakAfterJavaFieldAnnotations: false
BreakStringLiterals: false
ColumnLimit: 100
CommentPragmas: '^ IWYU pragma:'
CompactNamespaces: false
ConstructorInitializerAllOnOneLineOrOnePerLine: false
ConstructorInitializerIndentWidth : 8
ContinuationIndentWidth: 8
Cpp11BracedListStyle: false
DerivePointerAlignment: false
ExperimentalAutoDetectBinPacking: false
FixNamespaceComments: false
ForEachMacros:
- '__ata_qc_for_each'
- '__bio_for_each_bvec'
- '__bio_for_each_segment'
- '__evlist__for_each_entry'
- '__evlist__for_each_entry_continue'
- '__evlist__for_each_entry_from'
- '__evlist__for_each_entry_reverse'
- '__evlist__for_each_entry_safe'
- '__for_each_mem_range'
- '__for_each_mem_range_rev'
- '__for_each_thread'
- '__hlist_for_each_rcu'
- '__map__for_each_symbol_by_name'
- '__pci_bus_for_each_res0'
- '__pci_bus_for_each_res1'
- '__pci_dev_for_each_res0'
- '__pci_dev_for_each_res1'
- '__perf_evlist__for_each_entry'
- '__perf_evlist__for_each_entry_reverse'
- '__perf_evlist__for_each_entry_safe'
- '__rq_for_each_bio'
- '__shost_for_each_device'
- '__sym_for_each'
- 'apei_estatus_for_each_section'
- 'ata_for_each_dev'
- 'ata_for_each_link'
- 'ata_qc_for_each'
- 'ata_qc_for_each_raw'
- 'ata_qc_for_each_with_internal'
- 'ax25_for_each'
- 'ax25_uid_for_each'
- 'bio_for_each_bvec'
- 'bio_for_each_bvec_all'
- 'bio_for_each_folio_all'
- 'bio_for_each_integrity_vec'
- 'bio_for_each_segment'
- 'bio_for_each_segment_all'
- 'bio_list_for_each'
- 'bip_for_each_vec'
- 'bond_for_each_slave'
- 'bond_for_each_slave_rcu'
- 'bpf_for_each'
- 'bpf_for_each_reg_in_vstate'
- 'bpf_for_each_reg_in_vstate_mask'
- 'bpf_for_each_spilled_reg'
- 'bpf_object__for_each_map'
- 'bpf_object__for_each_program'
- 'btree_for_each_safe128'
- 'btree_for_each_safe32'
- 'btree_for_each_safe64'
- 'btree_for_each_safel'
- 'card_for_each_dev'
- 'cgroup_taskset_for_each'
- 'cgroup_taskset_for_each_leader'
- 'cpu_aggr_map__for_each_idx'
- 'cpufreq_for_each_efficient_entry_idx'
- 'cpufreq_for_each_entry'
- 'cpufreq_for_each_entry_idx'
- 'cpufreq_for_each_valid_entry'
- 'cpufreq_for_each_valid_entry_idx'
- 'css_for_each_child'
- 'css_for_each_descendant_post'
- 'css_for_each_descendant_pre'
- 'damon_for_each_region'
- 'damon_for_each_region_from'
- 'damon_for_each_region_safe'
- 'damon_for_each_scheme'
- 'damon_for_each_scheme_safe'
- 'damon_for_each_target'
- 'damon_for_each_target_safe'
- 'damos_for_each_filter'
- 'damos_for_each_filter_safe'
- 'damos_for_each_quota_goal'
- 'damos_for_each_quota_goal_safe'
- 'data__for_each_file'
- 'data__for_each_file_new'
- 'data__for_each_file_start'
- 'device_for_each_child_node'
- 'device_for_each_child_node_scoped'
- 'dma_fence_array_for_each'
- 'dma_fence_chain_for_each'
- 'dma_fence_unwrap_for_each'
- 'dma_resv_for_each_fence'
- 'dma_resv_for_each_fence_unlocked'
- 'do_for_each_ftrace_op'
- 'drm_atomic_crtc_for_each_plane'
- 'drm_atomic_crtc_state_for_each_plane'
- 'drm_atomic_crtc_state_for_each_plane_state'
- 'drm_atomic_for_each_plane_damage'
- 'drm_client_for_each_connector_iter'
- 'drm_client_for_each_modeset'
- 'drm_connector_for_each_possible_encoder'
- 'drm_exec_for_each_locked_object'
- 'drm_exec_for_each_locked_object_reverse'
- 'drm_for_each_bridge_in_chain'
- 'drm_for_each_connector_iter'
- 'drm_for_each_crtc'
- 'drm_for_each_crtc_reverse'
- 'drm_for_each_encoder'
- 'drm_for_each_encoder_mask'
- 'drm_for_each_fb'
- 'drm_for_each_legacy_plane'
- 'drm_for_each_plane'
- 'drm_for_each_plane_mask'
- 'drm_for_each_privobj'
- 'drm_gem_for_each_gpuvm_bo'
- 'drm_gem_for_each_gpuvm_bo_safe'
- 'drm_gpuva_for_each_op'
- 'drm_gpuva_for_each_op_from_reverse'
- 'drm_gpuva_for_each_op_reverse'
- 'drm_gpuva_for_each_op_safe'
- 'drm_gpuvm_bo_for_each_va'
- 'drm_gpuvm_bo_for_each_va_safe'
- 'drm_gpuvm_for_each_va'
- 'drm_gpuvm_for_each_va_range'
- 'drm_gpuvm_for_each_va_range_safe'
- 'drm_gpuvm_for_each_va_safe'
- 'drm_mm_for_each_hole'
- 'drm_mm_for_each_node'
- 'drm_mm_for_each_node_in_range'
- 'drm_mm_for_each_node_safe'
- 'dsa_switch_for_each_available_port'
- 'dsa_switch_for_each_cpu_port'
- 'dsa_switch_for_each_cpu_port_continue_reverse'
- 'dsa_switch_for_each_port'
- 'dsa_switch_for_each_port_continue_reverse'
- 'dsa_switch_for_each_port_safe'
- 'dsa_switch_for_each_user_port'
- 'dsa_switch_for_each_user_port_continue_reverse'
- 'dsa_tree_for_each_cpu_port'
- 'dsa_tree_for_each_user_port'
- 'dsa_tree_for_each_user_port_continue_reverse'
- 'dso__for_each_symbol'
- 'elf_hash_for_each_possible'
- 'elf_symtab__for_each_symbol'
- 'evlist__for_each_cpu'
- 'evlist__for_each_entry'
- 'evlist__for_each_entry_continue'
- 'evlist__for_each_entry_from'
- 'evlist__for_each_entry_reverse'
- 'evlist__for_each_entry_safe'
- 'flow_action_for_each'
- 'for_each_acpi_consumer_dev'
- 'for_each_acpi_dev_match'
- 'for_each_active_dev_scope'
- 'for_each_active_drhd_unit'
- 'for_each_active_iommu'
- 'for_each_active_route'
- 'for_each_aggr_pgid'
- 'for_each_and_bit'
- 'for_each_andnot_bit'
- 'for_each_available_child_of_node'
- 'for_each_available_child_of_node_scoped'
- 'for_each_bench'
- 'for_each_bio'
- 'for_each_board_func_rsrc'
- 'for_each_btf_ext_rec'
- 'for_each_btf_ext_sec'
- 'for_each_bvec'
- 'for_each_card_auxs'
- 'for_each_card_auxs_safe'
- 'for_each_card_components'
- 'for_each_card_dapms'
- 'for_each_card_pre_auxs'
- 'for_each_card_prelinks'
- 'for_each_card_rtds'
- 'for_each_card_rtds_safe'
- 'for_each_card_widgets'
- 'for_each_card_widgets_safe'
- 'for_each_cgroup_storage_type'
- 'for_each_child_of_node'
- 'for_each_child_of_node_scoped'
- 'for_each_clear_bit'
- 'for_each_clear_bit_from'
- 'for_each_clear_bitrange'
- 'for_each_clear_bitrange_from'
- 'for_each_cmd'
- 'for_each_cmsghdr'
- 'for_each_collection'
- 'for_each_comp_order'
- 'for_each_compatible_node'
- 'for_each_component_dais'
- 'for_each_component_dais_safe'
- 'for_each_conduit'
- 'for_each_console'
- 'for_each_console_srcu'
- 'for_each_cpu'
- 'for_each_cpu_and'
- 'for_each_cpu_andnot'
- 'for_each_cpu_from'
- 'for_each_cpu_or'
- 'for_each_cpu_wrap'
- 'for_each_dapm_widgets'
- 'for_each_dedup_cand'
- 'for_each_dev_addr'
- 'for_each_dev_scope'
- 'for_each_dma_cap_mask'
- 'for_each_dpcm_be'
- 'for_each_dpcm_be_rollback'
- 'for_each_dpcm_be_safe'
- 'for_each_dpcm_fe'
- 'for_each_drhd_unit'
- 'for_each_dss_dev'
- 'for_each_efi_memory_desc'
- 'for_each_efi_memory_desc_in_map'
- 'for_each_element'
- 'for_each_element_extid'
- 'for_each_element_id'
- 'for_each_enabled_cpu'
- 'for_each_endpoint_of_node'
- 'for_each_event'
- 'for_each_event_tps'
- 'for_each_evictable_lru'
- 'for_each_fib6_node_rt_rcu'
- 'for_each_fib6_walker_rt'
- 'for_each_file_lock'
- 'for_each_free_mem_pfn_range_in_zone_from'
- 'for_each_free_mem_range'
- 'for_each_free_mem_range_reverse'
- 'for_each_func_rsrc'
- 'for_each_gpiochip_node'
- 'for_each_group_evsel'
- 'for_each_group_evsel_head'
- 'for_each_group_member'
- 'for_each_group_member_head'
- 'for_each_hstate'
- 'for_each_hwgpio'
- 'for_each_if'
- 'for_each_inject_fn'
- 'for_each_insn'
- 'for_each_insn_op_loc'
- 'for_each_insn_prefix'
- 'for_each_intid'
- 'for_each_iommu'
- 'for_each_ip_tunnel_rcu'
- 'for_each_irq_nr'
- 'for_each_lang'
- 'for_each_link_ch_maps'
- 'for_each_link_codecs'
- 'for_each_link_cpus'
- 'for_each_link_platforms'
- 'for_each_lru'
- 'for_each_matching_node'
- 'for_each_matching_node_and_match'
- 'for_each_media_entity_data_link'
- 'for_each_mem_pfn_range'
- 'for_each_mem_range'
- 'for_each_mem_range_rev'
- 'for_each_mem_region'
- 'for_each_member'
- 'for_each_memory'
- 'for_each_migratetype_order'
- 'for_each_missing_reg'
- 'for_each_mle_subelement'
- 'for_each_mod_mem_type'
- 'for_each_net'
- 'for_each_net_continue_reverse'
- 'for_each_net_rcu'
- 'for_each_netdev'
- 'for_each_netdev_continue'
- 'for_each_netdev_continue_rcu'
- 'for_each_netdev_continue_reverse'
- 'for_each_netdev_dump'
- 'for_each_netdev_feature'
- 'for_each_netdev_in_bond_rcu'
- 'for_each_netdev_rcu'
- 'for_each_netdev_reverse'
- 'for_each_netdev_safe'
- 'for_each_new_connector_in_state'
- 'for_each_new_crtc_in_state'
- 'for_each_new_mst_mgr_in_state'
- 'for_each_new_plane_in_state'
- 'for_each_new_plane_in_state_reverse'
- 'for_each_new_private_obj_in_state'
- 'for_each_new_reg'
- 'for_each_nhlt_endpoint'
- 'for_each_nhlt_endpoint_fmtcfg'
- 'for_each_nhlt_fmtcfg'
- 'for_each_node'
- 'for_each_node_by_name'
- 'for_each_node_by_type'
- 'for_each_node_mask'
- 'for_each_node_state'
- 'for_each_node_with_cpus'
- 'for_each_node_with_property'
- 'for_each_nonreserved_multicast_dest_pgid'
- 'for_each_numa_hop_mask'
- 'for_each_of_allnodes'
- 'for_each_of_allnodes_from'
- 'for_each_of_cpu_node'
- 'for_each_of_pci_range'
- 'for_each_old_connector_in_state'
- 'for_each_old_crtc_in_state'
- 'for_each_old_mst_mgr_in_state'
- 'for_each_old_plane_in_state'
- 'for_each_old_private_obj_in_state'
- 'for_each_oldnew_connector_in_state'
- 'for_each_oldnew_crtc_in_state'
- 'for_each_oldnew_mst_mgr_in_state'
- 'for_each_oldnew_plane_in_state'
- 'for_each_oldnew_plane_in_state_reverse'
- 'for_each_oldnew_private_obj_in_state'
- 'for_each_online_cpu'
- 'for_each_online_node'
- 'for_each_online_pgdat'
- 'for_each_or_bit'
- 'for_each_path'
- 'for_each_pci_bridge'
- 'for_each_pci_dev'
- 'for_each_pcm_streams'
- 'for_each_physmem_range'
- 'for_each_populated_zone'
- 'for_each_possible_cpu'
- 'for_each_present_blessed_reg'
- 'for_each_present_cpu'
- 'for_each_prime_number'
- 'for_each_prime_number_from'
- 'for_each_probe_cache_entry'
- 'for_each_process'
- 'for_each_process_thread'
- 'for_each_prop_codec_conf'
- 'for_each_prop_dai_codec'
- 'for_each_prop_dai_cpu'
- 'for_each_prop_dlc_codecs'
- 'for_each_prop_dlc_cpus'
- 'for_each_prop_dlc_platforms'
- 'for_each_property_of_node'
- 'for_each_reg'
- 'for_each_reg_filtered'
- 'for_each_reloc'
- 'for_each_reloc_from'
- 'for_each_requested_gpio'
- 'for_each_requested_gpio_in_range'
- 'for_each_reserved_child_of_node'
- 'for_each_reserved_mem_range'
- 'for_each_reserved_mem_region'
- 'for_each_rtd_ch_maps'
- 'for_each_rtd_codec_dais'
- 'for_each_rtd_components'
- 'for_each_rtd_cpu_dais'
- 'for_each_rtd_dais'
- 'for_each_rtd_dais_reverse'
- 'for_each_sband_iftype_data'
- 'for_each_script'
- 'for_each_sec'
- 'for_each_set_bit'
- 'for_each_set_bit_from'
- 'for_each_set_bit_wrap'
- 'for_each_set_bitrange'
- 'for_each_set_bitrange_from'
- 'for_each_set_clump8'
- 'for_each_sg'
- 'for_each_sg_dma_page'
- 'for_each_sg_page'
- 'for_each_sgtable_dma_page'
- 'for_each_sgtable_dma_sg'
- 'for_each_sgtable_page'
- 'for_each_sgtable_sg'
- 'for_each_sibling_event'
- 'for_each_sta_active_link'
- 'for_each_subelement'
- 'for_each_subelement_extid'
- 'for_each_subelement_id'
- 'for_each_sublist'
- 'for_each_subsystem'
- 'for_each_supported_activate_fn'
- 'for_each_supported_inject_fn'
- 'for_each_sym'
- 'for_each_test'
- 'for_each_thread'
- 'for_each_token'
- 'for_each_unicast_dest_pgid'
- 'for_each_valid_link'
- 'for_each_vif_active_link'
- 'for_each_vma'
- 'for_each_vma_range'
- 'for_each_vsi'
- 'for_each_wakeup_source'
- 'for_each_zone'
- 'for_each_zone_zonelist'
- 'for_each_zone_zonelist_nodemask'
- 'func_for_each_insn'
- 'fwnode_for_each_available_child_node'
- 'fwnode_for_each_child_node'
- 'fwnode_for_each_parent_node'
- 'fwnode_graph_for_each_endpoint'
- 'gadget_for_each_ep'
- 'genradix_for_each'
- 'genradix_for_each_from'
- 'genradix_for_each_reverse'
- 'hash_for_each'
- 'hash_for_each_possible'
- 'hash_for_each_possible_rcu'
- 'hash_for_each_possible_rcu_notrace'
- 'hash_for_each_possible_safe'
- 'hash_for_each_rcu'
- 'hash_for_each_safe'
- 'hashmap__for_each_entry'
- 'hashmap__for_each_entry_safe'
- 'hashmap__for_each_key_entry'
- 'hashmap__for_each_key_entry_safe'
- 'hctx_for_each_ctx'
- 'hists__for_each_format'
- 'hists__for_each_sort_list'
- 'hlist_bl_for_each_entry'
- 'hlist_bl_for_each_entry_rcu'
- 'hlist_bl_for_each_entry_safe'
- 'hlist_for_each'
- 'hlist_for_each_entry'
- 'hlist_for_each_entry_continue'
- 'hlist_for_each_entry_continue_rcu'
- 'hlist_for_each_entry_continue_rcu_bh'
- 'hlist_for_each_entry_from'
- 'hlist_for_each_entry_from_rcu'
- 'hlist_for_each_entry_rcu'
- 'hlist_for_each_entry_rcu_bh'
- 'hlist_for_each_entry_rcu_notrace'
- 'hlist_for_each_entry_safe'
- 'hlist_for_each_entry_srcu'
- 'hlist_for_each_safe'
- 'hlist_nulls_for_each_entry'
- 'hlist_nulls_for_each_entry_from'
- 'hlist_nulls_for_each_entry_rcu'
- 'hlist_nulls_for_each_entry_safe'
- 'i3c_bus_for_each_i2cdev'
- 'i3c_bus_for_each_i3cdev'
- 'idr_for_each_entry'
- 'idr_for_each_entry_continue'
- 'idr_for_each_entry_continue_ul'
- 'idr_for_each_entry_ul'
- 'in_dev_for_each_ifa_rcu'
- 'in_dev_for_each_ifa_rtnl'
- 'inet_bind_bucket_for_each'
- 'interval_tree_for_each_span'
- 'intlist__for_each_entry'
- 'intlist__for_each_entry_safe'
- 'kcore_copy__for_each_phdr'
- 'key_for_each'
- 'key_for_each_safe'
- 'klp_for_each_func'
- 'klp_for_each_func_safe'
- 'klp_for_each_func_static'
- 'klp_for_each_object'
- 'klp_for_each_object_safe'
- 'klp_for_each_object_static'
- 'kunit_suite_for_each_test_case'
- 'kvm_for_each_memslot'
- 'kvm_for_each_memslot_in_gfn_range'
- 'kvm_for_each_vcpu'
- 'libbpf_nla_for_each_attr'
- 'list_for_each'
- 'list_for_each_codec'
- 'list_for_each_codec_safe'
- 'list_for_each_continue'
- 'list_for_each_entry'
- 'list_for_each_entry_continue'
- 'list_for_each_entry_continue_rcu'
- 'list_for_each_entry_continue_reverse'
- 'list_for_each_entry_from'
- 'list_for_each_entry_from_rcu'
- 'list_for_each_entry_from_reverse'
- 'list_for_each_entry_lockless'
- 'list_for_each_entry_rcu'
- 'list_for_each_entry_reverse'
- 'list_for_each_entry_safe'
- 'list_for_each_entry_safe_continue'
- 'list_for_each_entry_safe_from'
- 'list_for_each_entry_safe_reverse'
- 'list_for_each_entry_srcu'
- 'list_for_each_from'
- 'list_for_each_prev'
- 'list_for_each_prev_safe'
- 'list_for_each_rcu'
- 'list_for_each_reverse'
- 'list_for_each_safe'
- 'llist_for_each'
- 'llist_for_each_entry'
- 'llist_for_each_entry_safe'
- 'llist_for_each_safe'
- 'lwq_for_each_safe'
- 'map__for_each_symbol'
- 'map__for_each_symbol_by_name'
- 'mas_for_each'
- 'mci_for_each_dimm'
- 'media_device_for_each_entity'
- 'media_device_for_each_intf'
- 'media_device_for_each_link'
- 'media_device_for_each_pad'
- 'media_entity_for_each_pad'
- 'media_pipeline_for_each_entity'
- 'media_pipeline_for_each_pad'
- 'mlx5_lag_for_each_peer_mdev'
- 'msi_domain_for_each_desc'
- 'msi_for_each_desc'
- 'mt_for_each'
- 'nanddev_io_for_each_page'
- 'netdev_for_each_lower_dev'
- 'netdev_for_each_lower_private'
- 'netdev_for_each_lower_private_rcu'
- 'netdev_for_each_mc_addr'
- 'netdev_for_each_synced_mc_addr'
- 'netdev_for_each_synced_uc_addr'
- 'netdev_for_each_uc_addr'
- 'netdev_for_each_upper_dev_rcu'
- 'netdev_hw_addr_list_for_each'
- 'nft_rule_for_each_expr'
- 'nla_for_each_attr'
- 'nla_for_each_attr_type'
- 'nla_for_each_nested'
- 'nla_for_each_nested_type'
- 'nlmsg_for_each_attr'
- 'nlmsg_for_each_msg'
- 'nr_neigh_for_each'
- 'nr_neigh_for_each_safe'
- 'nr_node_for_each'
- 'nr_node_for_each_safe'
- 'of_for_each_phandle'
- 'of_property_for_each_string'
- 'of_property_for_each_u32'
- 'pci_bus_for_each_resource'
- 'pci_dev_for_each_resource'
- 'pcl_for_each_chunk'
- 'pcl_for_each_segment'
- 'pcm_for_each_format'
- 'perf_config_items__for_each_entry'
- 'perf_config_sections__for_each_entry'
- 'perf_config_set__for_each_entry'
- 'perf_cpu_map__for_each_cpu'
- 'perf_cpu_map__for_each_cpu_skip_any'
- 'perf_cpu_map__for_each_idx'
- 'perf_evlist__for_each_entry'
- 'perf_evlist__for_each_entry_reverse'
- 'perf_evlist__for_each_entry_safe'
- 'perf_evlist__for_each_evsel'
- 'perf_evlist__for_each_mmap'
- 'perf_hpp_list__for_each_format'
- 'perf_hpp_list__for_each_format_safe'
- 'perf_hpp_list__for_each_sort_list'
- 'perf_hpp_list__for_each_sort_list_safe'
- 'perf_tool_event__for_each_event'
- 'plist_for_each'
- 'plist_for_each_continue'
- 'plist_for_each_entry'
- 'plist_for_each_entry_continue'
- 'plist_for_each_entry_safe'
- 'plist_for_each_safe'
- 'pnp_for_each_card'
- 'pnp_for_each_dev'
- 'protocol_for_each_card'
- 'protocol_for_each_dev'
- 'queue_for_each_hw_ctx'
- 'radix_tree_for_each_slot'
- 'radix_tree_for_each_tagged'
- 'rb_for_each'
- 'rbtree_postorder_for_each_entry_safe'
- 'rdma_for_each_block'
- 'rdma_for_each_port'
- 'rdma_umem_for_each_dma_block'
- 'resort_rb__for_each_entry'
- 'resource_list_for_each_entry'
- 'resource_list_for_each_entry_safe'
- 'rhl_for_each_entry_rcu'
- 'rhl_for_each_rcu'
- 'rht_for_each'
- 'rht_for_each_entry'
- 'rht_for_each_entry_from'
- 'rht_for_each_entry_rcu'
- 'rht_for_each_entry_rcu_from'
- 'rht_for_each_entry_safe'
- 'rht_for_each_from'
- 'rht_for_each_rcu'
- 'rht_for_each_rcu_from'
- 'rq_for_each_bvec'
- 'rq_for_each_segment'
- 'rq_list_for_each'
- 'rq_list_for_each_safe'
- 'sample_read_group__for_each'
- 'scsi_for_each_prot_sg'
- 'scsi_for_each_sg'
- 'sctp_for_each_hentry'
- 'sctp_skb_for_each'
- 'sec_for_each_insn'
- 'sec_for_each_insn_continue'
- 'sec_for_each_insn_from'
- 'sec_for_each_sym'
- 'shdma_for_each_chan'
- 'shost_for_each_device'
- 'sk_for_each'
- 'sk_for_each_bound'
- 'sk_for_each_entry_offset_rcu'
- 'sk_for_each_from'
- 'sk_for_each_rcu'
- 'sk_for_each_safe'
- 'sk_nulls_for_each'
- 'sk_nulls_for_each_from'
- 'sk_nulls_for_each_rcu'
- 'snd_array_for_each'
- 'snd_pcm_group_for_each_entry'
- 'snd_soc_dapm_widget_for_each_path'
- 'snd_soc_dapm_widget_for_each_path_safe'
- 'snd_soc_dapm_widget_for_each_sink_path'
- 'snd_soc_dapm_widget_for_each_source_path'
- 'sparsebit_for_each_set_range'
- 'strlist__for_each_entry'
- 'strlist__for_each_entry_safe'
- 'sym_for_each_insn'
- 'sym_for_each_insn_continue_reverse'
- 'symbols__for_each_entry'
- 'tb_property_for_each'
- 'tcf_act_for_each_action'
- 'tcf_exts_for_each_action'
- 'ttm_resource_manager_for_each_res'
- 'udp_portaddr_for_each_entry'
- 'udp_portaddr_for_each_entry_rcu'
- 'usb_hub_for_each_child'
- 'v4l2_device_for_each_subdev'
- 'v4l2_m2m_for_each_dst_buf'
- 'v4l2_m2m_for_each_dst_buf_safe'
- 'v4l2_m2m_for_each_src_buf'
- 'v4l2_m2m_for_each_src_buf_safe'
- 'virtio_device_for_each_vq'
- 'while_for_each_ftrace_op'
- 'xa_for_each'
- 'xa_for_each_marked'
- 'xa_for_each_range'
- 'xa_for_each_start'
- 'xas_for_each'
- 'xas_for_each_conflict'
- 'xas_for_each_marked'
- 'xbc_array_for_each_value'
- 'xbc_for_each_key_value'
- 'xbc_node_for_each_array_value'
- 'xbc_node_for_each_child'
- 'xbc_node_for_each_key_value'
- 'xbc_node_for_each_subkey'
- 'ynl_attr_for_each'
- 'ynl_attr_for_each_nested'
- 'ynl_attr_for_each_payload'
- 'zorro_for_each_dev'
IncludeBlocks: Preserve
IncludeCategories:
- Regex: '.*'
Priority: 1
IncludeIsMainRegex: (Test)?$
IndentCaseLabels: false
IndentGotoLabels: false
IndentPPDirectives: None
IndentWidth: 8
IndentWrappedFunctionNames: false
JavaScriptQuotes: Leave
JavaScriptWrapImports: true
KeepEmptyLinesAtTheStartOfBlocks: false
MacroBlockBegin: ''
MacroBlockEnd: ''
MaxEmptyLinesToKeep: 1
NamespaceIndentation: None
ObjCBinPackProtocolList: Auto
ObjCBlockIndentWidth: 8
ObjCSpaceAfterProperty: true
ObjCSpaceBeforeProtocolList: true
PenaltyBreakAssignment: 10
PenaltyBreakBeforeFirstCallParameter: 30
PenaltyBreakComment: 10
PenaltyBreakFirstLessLess: 0
PenaltyBreakString: 10
PenaltyExcessCharacter: 100
PenaltyReturnTypeOnItsOwnLine: 60
PointerAlignment: Right
ReflowComments: false
SortIncludes: false
SortUsingDeclarations: false
SpaceAfterCStyleCast: false
SpaceAfterTemplateKeyword: true
SpaceBeforeAssignmentOperators: true
SpaceBeforeCtorInitializerColon: true
SpaceBeforeInheritanceColon: true
SpaceBeforeParens: ControlStatementsExceptForEachMacros
SpaceBeforeRangeBasedForLoopColon: true
SpaceInEmptyParentheses: false
SpacesBeforeTrailingComments: 1
SpacesInAngles: false
SpacesInContainerLiterals: false
SpacesInCStyleCastParentheses: false
SpacesInParentheses: false
SpacesInSquareBrackets: false
Standard: Cpp03
TabWidth: 8
UseTab: Never
...
+2
View File
@@ -0,0 +1,2 @@
CompileFlags:
CompilationDatabase: .
+138
View File
@@ -0,0 +1,138 @@
cmake_minimum_required(VERSION 3.16)
project(Nitload.Native LANGUAGES C)
include(${CMAKE_CURRENT_LIST_DIR}/ExternalLibs.cmake OPTIONAL)
set(SN_TARGET_OS "auto" CACHE STRING "Target OS identifier: auto, windows, linux, macos, ios, android, unknown")
set_property(CACHE SN_TARGET_OS PROPERTY STRINGS auto windows linux macos ios android unknown)
set(SN_TARGET_ARCH "auto" CACHE STRING "Target ISA identifier: auto, x86_64, arm64, riscv64, ppc64le")
set_property(CACHE SN_TARGET_ARCH PROPERTY STRINGS auto x86_64 arm64 riscv64 ppc64le)
sn_resolve_target_identifiers(
"${SN_TARGET_OS}"
"${SN_TARGET_ARCH}"
SN_RESOLVED_OS_NAME
SN_RESOLVED_OS_DEFINE
SN_RESOLVED_ARCH_NAME
SN_RESOLVED_ARCH_DEFINE)
file(GLOB STGSHARP_NATIVE_SOURCES CONFIGURE_DEPENDS
${CMAKE_CURRENT_LIST_DIR}/sn-src/*.c)
file(GLOB STGSHARP_NATIVE_HEADERS CONFIGURE_DEPENDS
${CMAKE_CURRENT_LIST_DIR}/sn-head/*.h)
file(GLOB STGSHARP_NATIVE_TEMPLATES CONFIGURE_DEPENDS
${CMAKE_CURRENT_LIST_DIR}/sn-template/*.h
${CMAKE_CURRENT_LIST_DIR}/sn-template/*.inl
${CMAKE_CURRENT_LIST_DIR}/sn-template/*.inc
${CMAKE_CURRENT_LIST_DIR}/sn-template/*.ipp)
if(NOT STGSHARP_NATIVE_SOURCES)
message(FATAL_ERROR "No source files found under sn-src/.")
endif()
# Nitload.Native now lives under <root>/src/Nitload.Native, so walk up two
# levels (project dir -> src -> solution root) to reach the solution root.
get_filename_component(STGSHARP_NATIVE_SRC_ROOT ${CMAKE_CURRENT_LIST_DIR} DIRECTORY)
get_filename_component(STGSHARP_SOLUTION_ROOT ${STGSHARP_NATIVE_SRC_ROOT} DIRECTORY)
if(SN_RESOLVED_OS_NAME STREQUAL "windows")
set(STGSHARP_OS_TAG win)
elseif(SN_RESOLVED_OS_NAME STREQUAL "linux")
set(STGSHARP_OS_TAG linux)
else()
set(STGSHARP_OS_TAG unknown)
endif()
if(SN_RESOLVED_ARCH_NAME STREQUAL "x86_64")
set(STGSHARP_ARCH_TAG x64)
else()
set(STGSHARP_ARCH_TAG ${SN_RESOLVED_ARCH_NAME})
endif()
set(STGSHARP_BUILD_OUTPUT_DIR ${STGSHARP_SOLUTION_ROOT}/bin/${STGSHARP_OS_TAG}-${STGSHARP_ARCH_TAG})
# Compile-time intermediates (native PDB / .obj side files) live inside the CMake
# build tree (cmake_build/<preset>) so they get cleaned up with it.
set(STGSHARP_OBJECT_ROOT_DIR ${CMAKE_CURRENT_BINARY_DIR}/obj/${PROJECT_NAME})
set(STGSHARP_NATIVE_PROJECT_FILES
${STGSHARP_NATIVE_SOURCES}
${STGSHARP_NATIVE_HEADERS}
${STGSHARP_NATIVE_TEMPLATES})
add_library(${PROJECT_NAME} SHARED ${STGSHARP_NATIVE_PROJECT_FILES})
source_group(TREE ${CMAKE_CURRENT_LIST_DIR} FILES ${STGSHARP_NATIVE_PROJECT_FILES})
set_target_properties(${PROJECT_NAME} PROPERTIES
C_STANDARD 11
C_STANDARD_REQUIRED ON
RUNTIME_OUTPUT_DIRECTORY ${STGSHARP_BUILD_OUTPUT_DIR}
LIBRARY_OUTPUT_DIRECTORY ${STGSHARP_BUILD_OUTPUT_DIR}
ARCHIVE_OUTPUT_DIRECTORY ${STGSHARP_BUILD_OUTPUT_DIR})
foreach(_config Debug Release RelWithDebInfo MinSizeRel)
string(TOUPPER ${_config} _config_upper)
set_target_properties(${PROJECT_NAME} PROPERTIES
RUNTIME_OUTPUT_DIRECTORY_${_config_upper} ${STGSHARP_BUILD_OUTPUT_DIR}
LIBRARY_OUTPUT_DIRECTORY_${_config_upper} ${STGSHARP_BUILD_OUTPUT_DIR}
ARCHIVE_OUTPUT_DIRECTORY_${_config_upper} ${STGSHARP_BUILD_OUTPUT_DIR}
PDB_OUTPUT_DIRECTORY_${_config_upper} ${STGSHARP_BUILD_OUTPUT_DIR}
COMPILE_PDB_OUTPUT_DIRECTORY_${_config_upper} ${STGSHARP_OBJECT_ROOT_DIR}/${_config}/${STGSHARP_ARCH_TAG})
endforeach()
set_target_properties(${PROJECT_NAME} PROPERTIES
COMPILE_PDB_OUTPUT_DIRECTORY ${STGSHARP_OBJECT_ROOT_DIR}/$<CONFIG>/${STGSHARP_ARCH_TAG})
if(MSVC OR (CMAKE_C_COMPILER_ID STREQUAL "Clang" AND CMAKE_C_COMPILER_FRONTEND_VARIANT STREQUAL "MSVC"))
target_compile_options(${PROJECT_NAME} PRIVATE
/Fd${STGSHARP_OBJECT_ROOT_DIR}/$<CONFIG>/${STGSHARP_ARCH_TAG}/)
endif()
if(CMAKE_C_COMPILER_ID STREQUAL "Clang" AND SN_RESOLVED_ARCH_NAME STREQUAL "x86_64")
# Keep the intrinsic type system and builtin declarations visible to all TUs.
# The runtime dispatch contract still follows T_arch/T_feature at the function level.
if(CMAKE_C_COMPILER_FRONTEND_VARIANT STREQUAL "MSVC")
target_compile_options(${PROJECT_NAME} PRIVATE
/clang:-mavx
/clang:-mavx2
/clang:-mfma
/clang:-mf16c
/clang:-mavx512f
/clang:-mavx512cd
/clang:-mavx512dq
/clang:-mavx512bw
/clang:-mavx512vl)
else()
target_compile_options(${PROJECT_NAME} PRIVATE
-mavx
-mavx2
-mfma
-mf16c
-mavx512f
-mavx512cd
-mavx512dq
-mavx512bw
-mavx512vl)
endif()
endif()
target_include_directories(${PROJECT_NAME}
PRIVATE
${CMAKE_CURRENT_LIST_DIR}/sn-head
${CMAKE_CURRENT_LIST_DIR}/sn-template
${CMAKE_CURRENT_LIST_DIR}/lib)
target_compile_definitions(${PROJECT_NAME}
PRIVATE
SN_EXTERN_TARGET
SN_OS_TARGET=${SN_RESOLVED_OS_DEFINE}
SN_ARCH_TARGET=${SN_RESOLVED_ARCH_DEFINE})
set(STGSHARP_NATIVE_EXTERNAL_TARGET ${PROJECT_NAME})
add_subdirectory(${CMAKE_CURRENT_LIST_DIR}/lib)
message(STATUS "Nitload.Native target OS: ${SN_RESOLVED_OS_NAME} (${SN_RESOLVED_OS_DEFINE})")
message(STATUS "Nitload.Native target ISA: ${SN_RESOLVED_ARCH_NAME} (${SN_RESOLVED_ARCH_DEFINE})")
message(STATUS "Nitload.Native output dir: ${STGSHARP_BUILD_OUTPUT_DIR}")
+51
View File
@@ -0,0 +1,51 @@
{
"version": 6,
"configurePresets": [
{
"name": "clang-debug",
"displayName": "Clang Debug",
"generator": "Ninja",
"binaryDir": "${sourceDir}/../../cmake_build/${presetName}",
"cacheVariables": {
"CMAKE_BUILD_TYPE": "Debug",
"CMAKE_EXPORT_COMPILE_COMMANDS": "ON",
"CMAKE_C_COMPILER": "clang",
"CMAKE_CXX_COMPILER": "clang++"
}
},
{
"name": "clang-release",
"displayName": "Clang Release",
"generator": "Ninja",
"binaryDir": "${sourceDir}/../../cmake_build/${presetName}",
"cacheVariables": {
"CMAKE_BUILD_TYPE": "Release",
"CMAKE_EXPORT_COMPILE_COMMANDS": "ON",
"CMAKE_C_COMPILER": "clang",
"CMAKE_CXX_COMPILER": "clang++"
}
},
{
"name": "使用工具链文件配置预设",
"displayName": "使用工具链文件配置预设",
"description": "设置 Ninja 生成器、版本和安装目录",
"generator": "Ninja",
"binaryDir": "${sourceDir}/../../cmake_build/${presetName}",
"cacheVariables": {
"CMAKE_BUILD_TYPE": "Debug",
"CMAKE_TOOLCHAIN_FILE": "",
"CMAKE_INSTALL_PREFIX": "${sourceDir}/out/install/${presetName}"
}
}
],
"buildPresets": [
{
"name": "clang-debug",
"configurePreset": "clang-debug"
},
{
"name": "clang-release",
"configurePreset": "clang-release"
}
]
}
+87
View File
@@ -0,0 +1,87 @@
function(sn_normalize_os_identifier INPUT_VALUE OUTPUT_NAME OUTPUT_DEFINE)
string(TOLOWER "${INPUT_VALUE}" _sn_os)
if(_sn_os STREQUAL "auto")
string(TOLOWER "${CMAKE_SYSTEM_NAME}" _sn_system_name)
if(_sn_system_name STREQUAL "windows")
set(_sn_os "windows")
elseif(_sn_system_name STREQUAL "linux")
set(_sn_os "linux")
endif()
endif()
if(_sn_os STREQUAL "windows")
set(_sn_define SN_OS_WINDOWS)
elseif(_sn_os STREQUAL "linux")
set(_sn_define SN_OS_LINUX)
else()
message(FATAL_ERROR "Unsupported SN_TARGET_OS='${INPUT_VALUE}'.")
endif()
set(${OUTPUT_NAME} "${_sn_os}" PARENT_SCOPE)
set(${OUTPUT_DEFINE} "${_sn_define}" PARENT_SCOPE)
endfunction()
function(sn_normalize_arch_identifier INPUT_VALUE OUTPUT_NAME OUTPUT_DEFINE)
string(TOLOWER "${INPUT_VALUE}" _sn_arch)
if(_sn_arch STREQUAL "auto")
string(TOLOWER "${CMAKE_SYSTEM_PROCESSOR}" _sn_processor)
if(_sn_processor MATCHES "^(amd64|x86_64|x64)$")
set(_sn_arch "x86_64")
elseif(_sn_processor MATCHES "^(arm64|aarch64)$")
set(_sn_arch "arm64")
elseif(_sn_processor STREQUAL "riscv64")
set(_sn_arch "riscv64")
elseif(_sn_processor MATCHES "^(ppc64le|powerpc64le)$")
set(_sn_arch "ppc64le")
else()
message(FATAL_ERROR "Unable to infer ISA from CMAKE_SYSTEM_PROCESSOR='${CMAKE_SYSTEM_PROCESSOR}'. Set SN_TARGET_ARCH explicitly.")
endif()
endif()
if(_sn_arch STREQUAL "x86_64")
set(_sn_define SN_ARCH_X86_64)
elseif(_sn_arch STREQUAL "arm64")
set(_sn_define SN_ARCH_ARM64)
elseif(_sn_arch STREQUAL "riscv64")
set(_sn_define SN_ARCH_RISCV64)
elseif(_sn_arch STREQUAL "ppc64le")
set(_sn_define SN_ARCH_PPC64le)
else()
message(FATAL_ERROR "Unsupported SN_TARGET_ARCH='${INPUT_VALUE}'.")
endif()
set(${OUTPUT_NAME} "${_sn_arch}" PARENT_SCOPE)
set(${OUTPUT_DEFINE} "${_sn_define}" PARENT_SCOPE)
endfunction()
function(sn_resolve_target_identifiers INPUT_OS INPUT_ARCH OUTPUT_OS_NAME OUTPUT_OS_DEFINE OUTPUT_ARCH_NAME OUTPUT_ARCH_DEFINE)
sn_normalize_os_identifier("${INPUT_OS}" _sn_os_name _sn_os_define)
sn_normalize_arch_identifier("${INPUT_ARCH}" _sn_arch_name _sn_arch_define)
set(${OUTPUT_OS_NAME} "${_sn_os_name}" PARENT_SCOPE)
set(${OUTPUT_OS_DEFINE} "${_sn_os_define}" PARENT_SCOPE)
set(${OUTPUT_ARCH_NAME} "${_sn_arch_name}" PARENT_SCOPE)
set(${OUTPUT_ARCH_DEFINE} "${_sn_arch_define}" PARENT_SCOPE)
endfunction()
function(sn_register_external_package TARGET PACKAGE_NAME PACKAGE_DIR)
if(NOT TARGET ${TARGET})
message(FATAL_ERROR "sn_register_external_package: target '${TARGET}' does not exist.")
endif()
if(EXISTS "${PACKAGE_DIR}")
target_include_directories(${TARGET} PRIVATE "${PACKAGE_DIR}")
message(STATUS "Registered external package '${PACKAGE_NAME}' include: ${PACKAGE_DIR}")
else()
message(FATAL_ERROR "External package directory not found: ${PACKAGE_DIR}")
endif()
endfunction()
function(sn_link_external_binary_if_exists TARGET LIBRARY_PATH)
if(EXISTS "${LIBRARY_PATH}")
target_link_libraries(${TARGET} PRIVATE "${LIBRARY_PATH}")
message(STATUS "Linked external binary: ${LIBRARY_PATH}")
endif()
endfunction()
+217
View File
@@ -0,0 +1,217 @@
<?xml version="1.0" encoding="utf-8"?>
<Project DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<!-- ════════════════════════════════════════════════════════ -->
<!-- ISA target:x64 + ARM64 -->
<!-- ════════════════════════════════════════════════════════ -->
<ItemGroup Label="ProjectConfigurations">
<ProjectConfiguration Include="Debug|x64">
<Configuration>Debug</Configuration>
<Platform>x64</Platform>
</ProjectConfiguration>
<ProjectConfiguration Include="Debug|ARM64">
<Configuration>Debug</Configuration>
<Platform>ARM64</Platform>
</ProjectConfiguration>
<ProjectConfiguration Include="Release|x64">
<Configuration>Release</Configuration>
<Platform>x64</Platform>
</ProjectConfiguration>
<ProjectConfiguration Include="Release|ARM64">
<Configuration>Release</Configuration>
<Platform>ARM64</Platform>
</ProjectConfiguration>
<ProjectConfiguration Include="PerformanceDebug|x64">
<Configuration>PerformanceDebug</Configuration>
<Platform>x64</Platform>
</ProjectConfiguration>
<ProjectConfiguration Include="PerformanceDebug|ARM64">
<Configuration>PerformanceDebug</Configuration>
<Platform>ARM64</Platform>
</ProjectConfiguration>
</ItemGroup>
<!-- ════════════════════════════════════════════════════════ -->
<!-- Global Property -->
<!-- ════════════════════════════════════════════════════════ -->
<PropertyGroup Label="Globals">
<VCProjectVersion>17.0</VCProjectVersion>
<Keyword>Win32Proj</Keyword>
<ProjectGuid>{20cbff26-6746-402e-992c-7b643e9423eb}</ProjectGuid>
<RootNamespace>NitloadC</RootNamespace>
<WindowsTargetPlatformVersion>10.0</WindowsTargetPlatformVersion>
<ProjectName>Nitload.Native</ProjectName>
</PropertyGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
<!-- ════════════════════════════════════════════════════════ -->
<!-- Configuration -->
<!-- ════════════════════════════════════════════════════════ -->
<!-- Debug(x64 + ARM64 共用) -->
<PropertyGroup Condition="'$(Configuration)'=='Debug'" Label="Configuration">
<ConfigurationType>DynamicLibrary</ConfigurationType>
<UseDebugLibraries>true</UseDebugLibraries>
<CharacterSet>Unicode</CharacterSet>
<WholeProgramOptimization>false</WholeProgramOptimization>
<PlatformToolset>v143</PlatformToolset>
</PropertyGroup>
<!-- Release(x64 + ARM64 共用) -->
<PropertyGroup Condition="'$(Configuration)'=='Release'" Label="Configuration">
<ConfigurationType>DynamicLibrary</ConfigurationType>
<UseDebugLibraries>false</UseDebugLibraries>
<WholeProgramOptimization>true</WholeProgramOptimization>
<CharacterSet>Unicode</CharacterSet>
<PlatformToolset>v143</PlatformToolset>
</PropertyGroup>
<!-- PerformanceDebug(x64 + ARM64 共用) -->
<PropertyGroup Condition="'$(Configuration)'=='PerformanceDebug'" Label="Configuration">
<ConfigurationType>DynamicLibrary</ConfigurationType>
<UseDebugLibraries>false</UseDebugLibraries>
<WholeProgramOptimization>true</WholeProgramOptimization>
<CharacterSet>Unicode</CharacterSet>
<PlatformToolset>v143</PlatformToolset>
</PropertyGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
<ImportGroup Label="ExtensionSettings">
</ImportGroup>
<ImportGroup Label="Shared">
</ImportGroup>
<!-- PropertySheets(对所有 Configuration|Platform 生效) -->
<ImportGroup Label="PropertySheets">
<Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
</ImportGroup>
<PropertyGroup Label="UserMacros" />
<!-- ════════════════════════════════════════════════════════ -->
<!-- output folder: layered -->
<!-- ════════════════════════════════════════════════════════ -->
<PropertyGroup>
<!-- 把 ARM64 → arm64,x64 保持 x64 -->
<_ArchTag Condition="'$(Platform)'=='x64'">x64</_ArchTag>
<_ArchTag Condition="'$(Platform)'=='ARM64'">arm64</_ArchTag>
<!-- 输出到 cmake_build/win_<arch>/,与 CMake 目录约定保持一致 -->
<OutDir>$(SolutionDir)cmake_build\win_$(_ArchTag)\</OutDir>
<IntDir>$(SolutionDir)obj\$(ProjectName)\$(Configuration)\$(Platform)\</IntDir>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)'=='Debug'">
<RunCodeAnalysis>true</RunCodeAnalysis>
<EnableClangTidyCodeAnalysis>false</EnableClangTidyCodeAnalysis>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)'=='Release' Or '$(Configuration)'=='PerformanceDebug'">
<GenerateManifest>false</GenerateManifest>
<RunCodeAnalysis>true</RunCodeAnalysis>
<EnableClangTidyCodeAnalysis>false</EnableClangTidyCodeAnalysis>
</PropertyGroup>
<!-- ════════════════════════════════════════════════════════ -->
<!-- Compile options: common to all configurations -->
<!-- ════════════════════════════════════════════════════════ -->
<ItemDefinitionGroup>
<ClCompile>
<WarningLevel>Level3</WarningLevel>
<SDLCheck>true</SDLCheck>
<ConformanceMode>true</ConformanceMode>
<LanguageStandard>Default</LanguageStandard>
<LanguageStandard_C>stdc11</LanguageStandard_C>
<MultiProcessorCompilation>true</MultiProcessorCompilation>
<FavorSizeOrSpeed>Speed</FavorSizeOrSpeed>
<AdditionalIncludeDirectories>$(ProjectDir)lib;$(ProjectDir)sn-head;$(ProjectDir)sn-template;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
</ClCompile>
<Link>
<!-- DLL 应该用 Windows,不是 Console -->
<SubSystem>Windows</SubSystem>
<AdditionalDependencies>$(CoreLibraryDependencies);%(AdditionalDependencies)</AdditionalDependencies>
</Link>
</ItemDefinitionGroup>
<!-- Debug 特有 -->
<ItemDefinitionGroup Condition="'$(Configuration)'=='Debug'">
<ClCompile>
<PreprocessorDefinitions>_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<InlineFunctionExpansion>Disabled</InlineFunctionExpansion>
<ExpandAttributedSource>true</ExpandAttributedSource>
<AssemblerOutput>All</AssemblerOutput>
<BrowseInformation>true</BrowseInformation>
</ClCompile>
<Link>
<GenerateDebugInformation>true</GenerateDebugInformation>
</Link>
<Bscmake>
<PreserveSbr>true</PreserveSbr>
</Bscmake>
</ItemDefinitionGroup>
<!-- Release + PerformanceDebug 共用(优化开启) -->
<ItemDefinitionGroup Condition="'$(Configuration)'=='Release' Or '$(Configuration)'=='PerformanceDebug'">
<ClCompile>
<PreprocessorDefinitions>NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<FunctionLevelLinking>true</FunctionLevelLinking>
<IntrinsicFunctions>true</IntrinsicFunctions>
<AssemblerOutput>All</AssemblerOutput>
<CompileAs>CompileAsC</CompileAs>
</ClCompile>
<Link>
<EnableCOMDATFolding>true</EnableCOMDATFolding>
<OptimizeReferences>true</OptimizeReferences>
<GenerateDebugInformation>false</GenerateDebugInformation>
<ShowProgress>LinkVerbose</ShowProgress>
<LinkTimeCodeGeneration>UseLinkTimeCodeGeneration</LinkTimeCodeGeneration>
</Link>
</ItemDefinitionGroup>
<!-- Release 独有(InlineFunctionExpansion) -->
<ItemDefinitionGroup Condition="'$(Configuration)'=='Release'">
<ClCompile>
<InlineFunctionExpansion>AnySuitable</InlineFunctionExpansion>
</ClCompile>
</ItemDefinitionGroup>
<!-- ════════════════════════════════════════════════════════ -->
<!-- glfw static lib -->
<!-- ════════════════════════════════════════════════════════ -->
<ItemGroup Condition="'$(Platform)'=='x64'">
<Library Include="lib\glfw\glfw3.lib" />
<Library Include="lib\glfw\glfw3_mt.lib" />
</ItemGroup>
<ItemGroup Condition="'$(Platform)'=='ARM64'">
<Library Include="lib\glfw\glfw3.lib" />
<Library Include="lib\glfw\glfw3_mt.lib" />
</ItemGroup>
<!-- ════════════════════════════════════════════════════════ -->
<!-- source files -->
<!-- ════════════════════════════════════════════════════════ -->
<ItemGroup>
<ClCompile Include="sn-src\dllmain.c" />
<ClCompile Include="sn-src\m.bufop.x86.c" />
<ClCompile Include="sn-src\m.solve.x86.c" />
<ClCompile Include="sn-src\m.transpose.x86.c" />
<ClCompile Include="sn-src\mk.fma.x86.c" />
<ClCompile Include="sn-src\mk.p_scan.x86.c" />
<ClCompile Include="sn-src\s.factorial.x86.c" />
<ClCompile Include="sn-src\sk.fma.x86.c" />
<ClCompile Include="sn-src\s.hash.x86.c" />
<ClCompile Include="sn-src\load_intrinsic.x86.c" />
<ClCompile Include="sn-src\load_glfw.c" />
<ClCompile Include="sn-src\simdid.x86.c" />
<ClCompile Include="sn-src\simdid_compare.x86.c" />
<ClCompile Include="sn-src\thread.linux.c" />
<ClCompile Include="sn-src\thread.windows.c" />
<ClCompile Include="sn-src\vector.c" />
<ClCompile Include="sn-src\vector_dot.c" />
</ItemGroup>
<!-- ════════════════════════════════════════════════════════ -->
<!-- header files -->
<!-- ════════════════════════════════════════════════════════ -->
<ItemGroup>
<ClInclude Include="lib\glfw\glfw3.h" />
<ClInclude Include="lib\glfw\glfw3native.h" />
<ClInclude Include="lib\stbi\stb_image.h" />
<ClInclude Include="sn-head\glfw_function.h" />
<ClInclude Include="sn-head\sn_internal.h" />
<ClInclude Include="sn-head\sn_intrinsic.context.x86.h" />
<ClInclude Include="sn-head\sn_intrinsic.std.h" />
<ClInclude Include="sn-head\sn_intrinsic.h" />
<ClInclude Include="sn-head\sn_intrinsic.matkernel.h" />
<ClInclude Include="sn-head\sn_target.h" />
<ClInclude Include="sn-head\StgSharpNative.h" />
<ClInclude Include="sn-template\sn_template.fma.x86.h" />
</ItemGroup>
<ItemGroup>
<None Include="naming.md" />
<None Include="SIMDID.x86.md" />
</ItemGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
<ImportGroup Label="ExtensionTargets">
</ImportGroup>
</Project>
@@ -0,0 +1,130 @@
<?xml version="1.0" encoding="utf-8"?>
<Project ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<ItemGroup>
<Filter Include="源文件">
<UniqueIdentifier>{4FC737F1-C7A5-4376-A066-2A32D752A2FF}</UniqueIdentifier>
<Extensions>cpp;c;cc;cxx;c++;cppm;ixx;def;odl;idl;hpj;bat;asm;asmx</Extensions>
</Filter>
<Filter Include="头文件">
<UniqueIdentifier>{93995380-89BD-4b04-88EB-625FBE52EBFB}</UniqueIdentifier>
<Extensions>h;hh;hpp;hxx;h++;hm;inl;inc;ipp;xsd</Extensions>
</Filter>
<Filter Include="资源文件">
<UniqueIdentifier>{67DA6AB6-F800-4c08-8B7A-83BB121AAD01}</UniqueIdentifier>
<Extensions>rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx;tiff;tif;png;wav;mfcribbon-ms</Extensions>
</Filter>
<Filter Include="Document">
<UniqueIdentifier>{fbd74b53-5314-4d9d-a788-07bd1b413f74}</UniqueIdentifier>
</Filter>
<Filter Include="模板文件">
<UniqueIdentifier>{c7081772-a85a-4372-b55c-a6dac43c6913}</UniqueIdentifier>
</Filter>
</ItemGroup>
<ItemGroup>
<ClCompile Include="sn-src\dllmain.c">
<Filter>源文件</Filter>
</ClCompile>
<ClCompile Include="sn-src\s.factorial.x86.c">
<Filter>源文件</Filter>
</ClCompile>
<ClCompile Include="sn-src\s.hash.x86.c">
<Filter>源文件</Filter>
</ClCompile>
<ClCompile Include="sn-src\vector.c">
<Filter>源文件</Filter>
</ClCompile>
<ClCompile Include="sn-src\vector_dot.c">
<Filter>源文件</Filter>
</ClCompile>
<ClCompile Include="sn-src\load_glfw.c">
<Filter>源文件</Filter>
</ClCompile>
<ClCompile Include="sn-src\sk.fma.x86.c">
<Filter>源文件</Filter>
</ClCompile>
<ClCompile Include="sn-src\load_intrinsic.x86.c">
<Filter>源文件</Filter>
</ClCompile>
<ClCompile Include="sn-src\m.bufop.x86.c">
<Filter>源文件</Filter>
</ClCompile>
<ClCompile Include="sn-src\m.solve.x86.c">
<Filter>源文件</Filter>
</ClCompile>
<ClCompile Include="sn-src\simdid.x86.c">
<Filter>源文件</Filter>
</ClCompile>
<ClCompile Include="sn-src\mk.fma.x86.c">
<Filter>源文件</Filter>
</ClCompile>
<ClCompile Include="sn-src\simdid_compare.x86.c">
<Filter>源文件</Filter>
</ClCompile>
<ClCompile Include="sn-src\thread.linux.c">
<Filter>源文件</Filter>
</ClCompile>
<ClCompile Include="sn-src\thread.windows.c">
<Filter>源文件</Filter>
</ClCompile>
<ClCompile Include="sn-src\m.transpose.x86.c">
<Filter>源文件</Filter>
</ClCompile>
<ClCompile Include="sn-src\mk.p_scan.x86.c">
<Filter>源文件</Filter>
</ClCompile>
</ItemGroup>
<ItemGroup>
<ClInclude Include="sn-head\sn_internal.h">
<Filter>头文件</Filter>
</ClInclude>
<ClInclude Include="sn-head\StgSharpNative.h">
<Filter>头文件</Filter>
</ClInclude>
<ClInclude Include="sn-head\glfw_function.h">
<Filter>头文件</Filter>
</ClInclude>
<ClInclude Include="sn-head\sn_intrinsic.matkernel.h">
<Filter>头文件</Filter>
</ClInclude>
<ClInclude Include="lib\glfw\glfw3.h">
<Filter>头文件</Filter>
</ClInclude>
<ClInclude Include="lib\glfw\glfw3native.h">
<Filter>头文件</Filter>
</ClInclude>
<ClInclude Include="lib\stbi\stb_image.h">
<Filter>头文件</Filter>
</ClInclude>
<ClInclude Include="sn-head\sn_target.h">
<Filter>头文件</Filter>
</ClInclude>
<ClInclude Include="sn-head\sn_intrinsic.context.x86.h">
<Filter>头文件</Filter>
</ClInclude>
<ClInclude Include="sn-head\sn_intrinsic.h">
<Filter>头文件</Filter>
</ClInclude>
<ClInclude Include="sn-head\sn_intrinsic.std.h">
<Filter>头文件</Filter>
</ClInclude>
<ClInclude Include="sn-template\sn_template.fma.x86.h">
<Filter>模板文件</Filter>
</ClInclude>
</ItemGroup>
<ItemGroup>
<Library Include="lib\glfw\glfw3.lib">
<Filter>资源文件</Filter>
</Library>
<Library Include="lib\glfw\glfw3_mt.lib">
<Filter>资源文件</Filter>
</Library>
</ItemGroup>
<ItemGroup>
<None Include="SIMDID.x86.md">
<Filter>Document</Filter>
</None>
<None Include="naming.md">
<Filter>Document</Filter>
</None>
</ItemGroup>
</Project>
+20
View File
@@ -0,0 +1,20 @@
# StgSharp.Native
## COMPILE
Though this project do has a .vcxproj file,
you can only edit but not compile in visul studio.
Currently you can only compile this lib via Clang and cmake.
## Code Layout
Conventions:
- src/ : project C sources and CMakeLists.txt calling add_library/add_executable
- include/ : public headers (if used)
- external/ : third-party libraries, each library lives in its own folder
- libname/
- head/ : header files for the library
- lib/ : static or shared lib files (or CMake build for that lib)
The root `CMakeLists.txt` in this folder includes `ExternalLibs.cmake` if present.
+258
View File
@@ -0,0 +1,258 @@
# SIMDID (x86-64) 64-bit Specification (Completed per fixed layout)
This document defines the compact **64-bit SIMDID** layout, including AVX/AVX-512/AVX10 capabilities and per-core execution-policy hints. AMX is not represented by the current policy.
> Convention: bit0 is the least significant bit (LSB). Only x86-64 interpreter semantics are defined here; the header's Root ISA allows extension to other architectures.
---
## 1) ASCII Bit Layout (Final)
```
SIMDID (64 bits) [63 ........................................................ 0]
+----------+-----------+-----------+----------------+----------------+----------------+--------+-------+
| [63..56] | [55..48] | [47..40] | [39..32] | [31..16] | [15..8] | [7..4] | [3..0]|
| Reserved | uArchHi | AVX10Feat | AVX Feature | AVX512 Feature | MainLevel | Manuf | Root |
+----------+-----------+-----------+----------------+----------------+----------------+--------+-------+
```
- **[3..0] Root ISA**: Root ISA type (selects interpreter)
- **[7..4] Manufacture**: Vendor info (Intel/AMD for x86)
- **[15..8] MainLevel**: Main ISA level (ascending, shared bit semantics)
- **[31..16] AVX512 Feature**: AVX-512 features/subsets/implementation form (key part)
- **[39..32] AVX Feature**: AVX-family capabilities (base requires AVX+AVX2; key extensions include FMA/F16C)
- **[47..40] AVX10 Feature**: AVX10 version and tracked discrete features
- **[55..48] uArchHi**: Optional microarchitecture/policy hints
- **[63..56] Reserved**: tail space for future extension
---
## 2) Root ISA ([3..0])
Suggested encoding (minimal usable set, expandable if needed):
| Value | Root ISA |
|---:|----------|
| 0 | Unknown |
| 1 | x86-64 |
| 2 | AArch64 |
| 3 | PPC64LE |
| 4 | RISC-V |
| 5 | LoongArch |
| 6..15 | Reserved |
> Note: Apple Silicon is architecturally AArch64 and uses `Root ISA = 2`. Apple vs Qualcomm vs other AArch64 vendors are distinguished via the **Manufacture** nibble under the AArch64 interpreter, not via a separate Root ISA value.
---
## 3) Manufacture ([7..4])
For **x86-64**:
| Value | Manufacture |
|---:|-------------|
| 0 | Unknown/Other |
| 1 | Intel |
| 2 | AMD |
| 3 | VIA/Zhaoxin |
| 4 | Hygon (海光) |
| 5..15 | Reserved |
For **AArch64**:
| Value | Manufacture |
|---:|-------------|
| 0 | Unknown/Other |
| 1 | Apple |
| 2 | Qualcomm |
| 3 | MediaTek |
| 4 | Ampere |
| 5..15 | Reserved |
For other Root ISAs, this nibble is defined by their interpreters.
---
## 4) MainLevel ([15..8], ascending)
"Shared bits and ascending" means higher values indicate higher capability (or later scheduling order). Suggested values:
| Value | MainLevel |
|---:|-----------|
| 0 | NONE |
| 1 | SSE (full SSE family: SSE1..SSE4.2) |
| 2 | AVX2 |
| 3 | AVX512 (check [31..16]) |
| 4 | AVX10 (check [47..40]) |
| 5..255 | Reserved |
> `MainLevel` is the primary switch. Parse AVX512 Feature at AVX512 or later and AVX10 Feature only at AVX10.
---
## 5) AVX512 Feature ([31..16]) Design Goals (Compact)
1) **Base AVX512 one-shot check**: Use 1 bit for the bundle `F + CD + VL + DQ + BW`, avoiding multiple subset checks.
2) **Glue vs native**: Use an implementation field to distinguish `AVX512-DUAL` (glue/downclock risk) from `AVX512-NATIVE` (full 512-bit ALU).
3) **Minimal extension tracking**: VNNI is the only AVX-512 extension currently retained.
---
## 6) AVX512 Feature Bit Layout ([31..16] = 16 bits, compact)
AVX512 Feature (bits 31..16)
| **bit** | 31 | 30 | 29 | 28 | 27 | 26 | 25 | 24 | 23 | 22 | 21 | 20 | 19 | 18 | 17 | 16 |
|-----------|-----|-----|------|------|------|-------|------|------|-------|-------|----|------|----|----|----|----|
| **Usage** | R | R | R | R | VNNI | R | R | R | Impl1 | Impl0 | R | BASE | R | R | R | R |
Notes:
- **BASE (bit 20)**: Base AVX512 bundle, indicating `F + CD + VL + DQ + BW` are all present. If set to 1, the detecting side must guarantee these subsets are all available.
- **AVX512_IMPL ([23..22], 2-bit)**: 512-bit execution width policy — not port count, not frequency behaviour.
| Value | Meaning |
|---:|------|
| 0 | Unknown — cannot determine; use conservative 256-bit kernels |
| 1 | DUAL-PUMP — single 256-bit ALU executes 512-bit ops as two consecutive 256-bit micro-ops; ZMM throughput halved (example: AMD Zen4, Family 0x19) |
| 2 | NATIVE — genuine 512-bit execution unit(s); full ZMM throughput (Intel all models including SKX; AMD Zen5 Family 0x1A and later) |
| 3 | Reserved |
> **Note**: Intel SKX/CLX/CPX have two 512-bit FMA ports (port-0 + port-5) and are **NATIVE** — genuine 512-bit throughput. Intel downclocking on heavy ZMM workloads is a separate concern not encoded here. DUAL-PUMP applies only where a single 256-bit unit fires twice per 512-bit op (AMD Zen4-style).
- **VNNI (bit 27)**: the only AVX-512 extension currently tracked; the surrounding extension bits are reserved.
- **R/AI (bit 24)**: Reserved for future AI or other non-crypto extensions.
- **[31..30], [21], [19..16]**: Reserved for future allocation.
---
## 7) AVX Feature Byte ([39..32], 8 bits)
Purpose: compact AVX-family capability encoding. Base requirement: AVX **and** AVX2 present. Key extensions: FMA and F16C.
AVX Feature (bits 39..32)
| bit | 39 | 38 | 37 | 36 | 35 | 34 | 33 | 32 |
|-----|----|----|----|----|----|----|----|----|
| use | R | R | R | DUAL | F16C | FMA | AVX2 | AVX |
Rules:
- **AVX (bit 32)**: AVX present.
- **AVX2 (bit 33)**: AVX2 present. For the AVX feature byte to be considered valid, both AVX and AVX2 must be set.
- **FMA (bit 34)**: FMA present.
- **F16C (bit 35)**: F16C present.
- **DUAL (bit 36)**: YMM execution is **glued/split-lane** (two fused 128-bit units rather than a native 256-bit path). Detection: AMD Zen1/Zen+ (Family 0x17, model < 0x30), Hygon (all, Zen1-based), VIA/Zhaoxin (all). When set, code using YMM operations may incur split-register overhead; prefer 128-bit SSE paths on these targets.
- **[37..39]**: Reserved for other AVX-family extensions as needed.
---
## 8) AVX10 Feature Byte ([47..40], 8 bits)
Purpose: encode the converged AVX10 ISA version and the discrete features relevant to this runtime. Under the current specification every AVX10 processor supports 128-, 256-, and 512-bit vector lengths, so vector width is not encoded.
**CPUID detection**: `CPUID leaf 0x24, subleaf 0`.
- EBX[7:0] = AVX10 converged ISA version number (≥ 1 if supported).
- EBX[18:16] are reserved at 1. Earlier specifications used them for VL128/VL256/VL512; they must not be interpreted as width capabilities.
- `CPUID.24H.1:ECX[2]` enumerates `AVX10_VNNI_INT` when subleaf 1 exists.
AVX10 Feature (bits 47..40)
| bit | 47 | 46..40 |
|-----|----------|---------|
| use | VNNI_INT | Version |
Rules:
- **Version ([46..40])**: numeric AVX10 version, saturated to 7 bits. Zero means AVX10 is absent.
- **VNNI_INT (bit 47)**: `CPUID.24H.1:ECX[2]`.
> AVX10 version comparison is numeric. Merging two SIMDIDs keeps the lower version and intersects discrete feature bits.
**Relationship with AVX512 Feature**:
- AVX10 is a superset specification. When `MainLevel == AVX10`, the `AVX512 Feature` field may still be populated for backward-compatible code paths (e.g., checking `AVX512_IMPL` to select DUAL vs NATIVE 512-bit policy).
- Current AVX10 hardware is architecturally AVX10/512. OS support still requires the full XCR0 SSE/AVX/opmask/ZMM state.
---
## 9) "AVX512BASE" Determination (Single Check)
- **Rule**: `BASE=1` implies `F+CD+VL+DQ+BW` are all present; no need for per-subset checks. Detection must keep this consistent; if not all subsets are present, set to 0.
- **Runtime branching**:
- If `MainLevel < AVX512`: fall back to AVX2/FMA path.
- Otherwise:
- Check `BASE`; if 0, fall back to AVX2/FMA (or treat as detection failure).
- Check `AVX512_IMPL`: `DUAL` uses 256-bit/downclock-safe path, `NATIVE` uses 512-bit path.
- Consult `VNNI` when the integer dot-product path requires it.
- If `MainLevel == AVX10`: prefer AVX10 Feature for capability checks; `AVX512 Feature` remains valid for legacy branching.
---
## 10) uArchHi ([55..48], 8 bits)
Purpose: **bitmask** byte encoding core topology classification and SIMD execution-width policy for the detected core. All bits default to 0.
uArchHi bitmask layout:
| bit (relative to [55..48]) | Name | Note |
|---:|------|------|
| 0 | HYBRID_E | E-core (efficiency/small core) in a hybrid-topology processor (e.g., Intel Alder Lake+ E-core) |
| 1 | HYBRID_P | P-core (performance/big core) in a hybrid-topology processor (e.g., Intel Alder Lake+ P-core) |
| 2 | SIMD_DUAL | SIMD execution is glued/dual (not full-width). Summary flag: set when `AVX_DUAL=1` **or** `AVX512_IMPL=DUAL`. |
| 3..7 | Reserved | |
Core topology encoding (bits [1..0], numerically comparable — higher = stronger):
| bits [1..0] | Value | Meaning |
|:-----------:|:-----:|----------|
| 00 | 0 | Unknown — undetected hybrid type or pre-CPUID.1A hardware |
| 01 | 1 | **Hybrid-E** — efficiency/small core (a3) |
| 10 | 2 | **Hybrid-P** — performance/big core (a2) |
| 11 | 3 | **Unified** — confirmed non-hybrid; full-topology core (a1) |
The 2-bit field is directly comparable as an unsigned integer: `unified(3) > P-core(2) > E-core(1) > unknown(0)`.
Detection:
- **Non-hybrid (unified)**: if `CPUID.7.0 EDX[15]` (HYBRID) is **not** set, the CPU is confirmed non-hybrid → set both `HYBRID_P` and `HYBRID_E` (topology = 11).
- **Hybrid P-core**: HYBRID flag set and `CPUID.0x1A.0 EAX[31:24] == 0x40` → set only `HYBRID_P` (topology = 10).
- **Hybrid E-core**: HYBRID flag set and `CPUID.0x1A.0 EAX[31:24] == 0x20` → set only `HYBRID_E` (topology = 01).
- **Unknown hybrid** (flag set but core type unrecognised): leave topology at 00.
- **SIMD_DUAL**: set when any of the following: `AVX Feature DUAL=1` (glued YMM), or `AVX512_IMPL=DUAL`.
`sn_get_unite_simd` computes topology bits [1..0] with bitwise **AND** (conservative intersection — weaker topology wins) and SIMD_DUAL with bitwise **OR** (dual if either core is dual). `sn_compare_simd` compares topology value numerically as step 6 and SIMD_DUAL as the final (step 7) tie-break.
---
## 12) Minimal Detection-Side Fill Rules (x86-64)
1) Set `RootISA = x86-64`, `Manufacture = Intel / AMD / Hygon / VIA`.
2) Compute `MainLevel` (per ascending rule).
3) If `MainLevel >= AVX512`:
- If `F+CD+VL+DQ+BW` are all present, set `BASE = 1`; otherwise `BASE = 0`.
- Set `AVX512_IMPL`:
- **Intel (all)**: NATIVE — all Intel AVX-512 CPUs have genuine 512-bit execution units
- **AMD Zen4 (Family 0x19)**: DUAL — single 256-bit ALU, dual-pump
- **AMD Family 0x1A models 00h..0Fh (EPYC 9005)**: NATIVE — full-width 512-bit path
- **AMD Family 0x1A models 10h..1Fh**: UNKNOWN — this CPUID range is shared by full-width EPYC 9005 Zen5c and 256-bit dual-pumped EPYC 8005 products
- **AMD other client/future models**: UNKNOWN unless their execution width can be identified without an ambiguous product-name heuristic
- **Hygon**: UNKNOWN (AVX-512 characteristics unclear)
- **VIA/other**: UNKNOWN
- Set VNNI when detected; other AVX-512 extension bits remain reserved.
4) When AVX/AVX2 are present, populate the AVX Feature byte: set `AVX` and `AVX2`; set `FMA` and `F16C` when detected; set `DUAL` when the YMM execution path is split/glued (AMD Zen1/Zen+ Family 0x17 model < 0x30, Hygon all models, VIA/Zhaoxin all models).
5) **AVX10 detection** (requires `CPUID leaf 7 subleaf 1, EDX[19] == 1` as the AVX10 convergence bit):
- If AVX10 is indicated, query `CPUID leaf 0x24, subleaf 0`:
- Read `EBX[7:0]` as the numeric version; do not interpret `EBX[18:16]` as vector-width bits.
- If subleaf 1 exists, read `ECX[2]` as `AVX10_VNNI_INT`.
- Promote `MainLevel` to `AVX10` (value 4).
- The `AVX512 Feature` field should still be filled for backward compatibility.
6) Populate uArchHi:
- If `CPUID.7.0 EDX[15]` (HYBRID) **not** set: set `HYBRID_P | HYBRID_E` (unified = 11).
- If HYBRID is set, read `CPUID.0x1A.0 EAX[31:24]`:
- `0x40` (P-core) → set only `HYBRID_P` (topology = 10).
- `0x20` (E-core) → set only `HYBRID_E` (topology = 01).
- Otherwise leave topology at 00 (unknown hybrid).
- If `AVX Feature DUAL=1` or `AVX512_IMPL=DUAL`, set `SIMD_DUAL`.
---
## 13) Choice for High Bits Reservation
`[31..24]` is reserved for AVX512 extensions/AI. `[39..37]` is reserved for AVX-family extensions. AVX10 `[47..40]` stores one tracked discrete feature plus a 7-bit version. uArchHi `[55..48]` bits [3..7] are reserved; bits [0..2] encode hybrid topology and SIMD-dual policy. `[63..56]` is reserved tail space.
+22
View File
@@ -0,0 +1,22 @@
cmake_minimum_required(VERSION 3.16)
if(NOT DEFINED STGSHARP_NATIVE_EXTERNAL_TARGET)
message(FATAL_ERROR "STGSHARP_NATIVE_EXTERNAL_TARGET must be set before adding lib/.")
endif()
include(${CMAKE_CURRENT_LIST_DIR}/../ExternalLibs.cmake OPTIONAL)
file(GLOB LIB_PACKAGE_DIRS RELATIVE ${CMAKE_CURRENT_LIST_DIR} ${CMAKE_CURRENT_LIST_DIR}/*)
foreach(_pkg ${LIB_PACKAGE_DIRS})
set(_pkg_dir ${CMAKE_CURRENT_LIST_DIR}/${_pkg})
if(NOT IS_DIRECTORY ${_pkg_dir})
continue()
endif()
if(EXISTS ${_pkg_dir}/CMakeLists.txt)
add_subdirectory(${_pkg_dir})
else()
sn_register_external_package(${STGSHARP_NATIVE_EXTERNAL_TARGET} ${_pkg} ${_pkg_dir})
endif()
endforeach()
@@ -0,0 +1,11 @@
cmake_minimum_required(VERSION 3.16)
sn_register_external_package(${STGSHARP_NATIVE_EXTERNAL_TARGET} glfw ${CMAKE_CURRENT_LIST_DIR})
if(WIN32)
if(NOT EXISTS ${CMAKE_CURRENT_LIST_DIR}/glfw3_mt.lib)
message(FATAL_ERROR "Required GLFW static library not found: ${CMAKE_CURRENT_LIST_DIR}/glfw3_mt.lib")
endif()
sn_link_external_binary_if_exists(${STGSHARP_NATIVE_EXTERNAL_TARGET} ${CMAKE_CURRENT_LIST_DIR}/glfw3_mt.lib)
endif()
File diff suppressed because it is too large Load Diff
Binary file not shown.
Binary file not shown.
Binary file not shown.
+663
View File
@@ -0,0 +1,663 @@
/* clang-format off */
/*************************************************************************
* GLFW 3.4 - www.glfw.org
* A library for OpenGL, window and input
*------------------------------------------------------------------------
* Copyright (c) 2002-2006 Marcus Geelnard
* Copyright (c) 2006-2018 Camilla Löwy <elmindreda@glfw.org>
*
* This software is provided 'as-is', without any express or implied
* warranty. In no event will the authors be held liable for any damages
* arising from the use of this software.
*
* Permission is granted to anyone to use this software for any purpose,
* including commercial applications, and to alter it and redistribute it
* freely, subject to the following restrictions:
*
* 1. The origin of this software must not be misrepresented; you must not
* claim that you wrote the original software. If you use this software
* in a product, an acknowledgment in the product documentation would
* be appreciated but is not required.
*
* 2. Altered source versions must be plainly marked as such, and must not
* be misrepresented as being the original software.
*
* 3. This notice may not be removed or altered from any source
* distribution.
*
*************************************************************************/
#ifndef _glfw3_native_h_
#define _glfw3_native_h_
#ifdef __cplusplus
extern "C" {
#endif
/*************************************************************************
* Doxygen documentation
*************************************************************************/
/*! @file glfw3native.h
* @brief The header of the native access functions.
*
* This is the header file of the native access functions. See @ref native for
* more information.
*/
/*! @defgroup native Native access
* @brief Functions related to accessing native handles.
*
* **By using the native access functions you assert that you know what you're
* doing and how to fix problems caused by using them. If you don't, you
* shouldn't be using them.**
*
* Before the inclusion of @ref glfw3native.h, you may define zero or more
* window system API macro and zero or more context creation API macros.
*
* The chosen backends must match those the library was compiled for. Failure
* to do this will cause a link-time error.
*
* The available window API macros are:
* * `GLFW_EXPOSE_NATIVE_WIN32`
* * `GLFW_EXPOSE_NATIVE_COCOA`
* * `GLFW_EXPOSE_NATIVE_X11`
* * `GLFW_EXPOSE_NATIVE_WAYLAND`
*
* The available context API macros are:
* * `GLFW_EXPOSE_NATIVE_WGL`
* * `GLFW_EXPOSE_NATIVE_NSGL`
* * `GLFW_EXPOSE_NATIVE_GLX`
* * `GLFW_EXPOSE_NATIVE_EGL`
* * `GLFW_EXPOSE_NATIVE_OSMESA`
*
* These macros select which of the native access functions that are declared
* and which platform-specific headers to include. It is then up your (by
* definition platform-specific) code to handle which of these should be
* defined.
*
* If you do not want the platform-specific headers to be included, define
* `GLFW_NATIVE_INCLUDE_NONE` before including the @ref glfw3native.h header.
*
* @code
* #define GLFW_EXPOSE_NATIVE_WIN32
* #define GLFW_EXPOSE_NATIVE_WGL
* #define GLFW_NATIVE_INCLUDE_NONE
* #include <GLFW/glfw3native.h>
* @endcode
*/
/*************************************************************************
* System headers and types
*************************************************************************/
#if !defined(GLFW_NATIVE_INCLUDE_NONE)
#if defined(GLFW_EXPOSE_NATIVE_WIN32) || defined(GLFW_EXPOSE_NATIVE_WGL)
/* This is a workaround for the fact that glfw3.h needs to export APIENTRY (for
* example to allow applications to correctly declare a GL_KHR_debug callback)
* but windows.h assumes no one will define APIENTRY before it does
*/
#if defined(GLFW_APIENTRY_DEFINED)
#undef APIENTRY
#undef GLFW_APIENTRY_DEFINED
#endif
#include <windows.h>
#endif
#if defined(GLFW_EXPOSE_NATIVE_COCOA) || defined(GLFW_EXPOSE_NATIVE_NSGL)
#if defined(__OBJC__)
#import <Cocoa/Cocoa.h>
#else
#include <ApplicationServices/ApplicationServices.h>
#include <objc/objc.h>
#endif
#endif
#if defined(GLFW_EXPOSE_NATIVE_X11) || defined(GLFW_EXPOSE_NATIVE_GLX)
#include <X11/Xlib.h>
#include <X11/extensions/Xrandr.h>
#endif
#if defined(GLFW_EXPOSE_NATIVE_WAYLAND)
#include <wayland-client.h>
#endif
#if defined(GLFW_EXPOSE_NATIVE_WGL)
/* WGL is declared by windows.h */
#endif
#if defined(GLFW_EXPOSE_NATIVE_NSGL)
/* NSGL is declared by Cocoa.h */
#endif
#if defined(GLFW_EXPOSE_NATIVE_GLX)
/* This is a workaround for the fact that glfw3.h defines GLAPIENTRY because by
* default it also acts as an OpenGL header
* However, glx.h will include gl.h, which will define it unconditionally
*/
#if defined(GLFW_GLAPIENTRY_DEFINED)
#undef GLAPIENTRY
#undef GLFW_GLAPIENTRY_DEFINED
#endif
#include <GL/glx.h>
#endif
#if defined(GLFW_EXPOSE_NATIVE_EGL)
#include <EGL/egl.h>
#endif
#if defined(GLFW_EXPOSE_NATIVE_OSMESA)
/* This is a workaround for the fact that glfw3.h defines GLAPIENTRY because by
* default it also acts as an OpenGL header
* However, osmesa.h will include gl.h, which will define it unconditionally
*/
#if defined(GLFW_GLAPIENTRY_DEFINED)
#undef GLAPIENTRY
#undef GLFW_GLAPIENTRY_DEFINED
#endif
#include <GL/osmesa.h>
#endif
#endif /*GLFW_NATIVE_INCLUDE_NONE*/
/*************************************************************************
* Functions
*************************************************************************/
#if defined(GLFW_EXPOSE_NATIVE_WIN32)
/*! @brief Returns the adapter device name of the specified monitor.
*
* @return The UTF-8 encoded adapter device name (for example `\\.\DISPLAY1`)
* of the specified monitor, or `NULL` if an [error](@ref error_handling)
* occurred.
*
* @errors Possible errors include @ref GLFW_NOT_INITIALIZED and @ref
* GLFW_PLATFORM_UNAVAILABLE.
*
* @thread_safety This function may be called from any thread. Access is not
* synchronized.
*
* @since Added in version 3.1.
*
* @ingroup native
*/
GLFWAPI const char *glfwGetWin32Adapter(GLFWmonitor *monitor);
/*! @brief Returns the display device name of the specified monitor.
*
* @return The UTF-8 encoded display device name (for example
* `\\.\DISPLAY1\Monitor0`) of the specified monitor, or `NULL` if an
* [error](@ref error_handling) occurred.
*
* @errors Possible errors include @ref GLFW_NOT_INITIALIZED and @ref
* GLFW_PLATFORM_UNAVAILABLE.
*
* @thread_safety This function may be called from any thread. Access is not
* synchronized.
*
* @since Added in version 3.1.
*
* @ingroup native
*/
GLFWAPI const char *glfwGetWin32Monitor(GLFWmonitor *monitor);
/*! @brief Returns the `HWND` of the specified window.
*
* @return The `HWND` of the specified window, or `NULL` if an
* [error](@ref error_handling) occurred.
*
* @errors Possible errors include @ref GLFW_NOT_INITIALIZED and @ref
* GLFW_PLATFORM_UNAVAILABLE.
*
* @remark The `HDC` associated with the window can be queried with the
* [GetDC](https://docs.microsoft.com/en-us/windows/win32/api/winuser/nf-winuser-getdc)
* function.
* @code
* HDC dc = GetDC(glfwGetWin32Window(window));
* @endcode
* This DC is private and does not need to be released.
*
* @thread_safety This function may be called from any thread. Access is not
* synchronized.
*
* @since Added in version 3.0.
*
* @ingroup native
*/
GLFWAPI HWND glfwGetWin32Window(GLFWwindow *window);
#endif
#if defined(GLFW_EXPOSE_NATIVE_WGL)
/*! @brief Returns the `HGLRC` of the specified window.
*
* @return The `HGLRC` of the specified window, or `NULL` if an
* [error](@ref error_handling) occurred.
*
* @errors Possible errors include @ref GLFW_NOT_INITIALIZED, @ref
* GLFW_PLATFORM_UNAVAILABLE and @ref GLFW_NO_WINDOW_CONTEXT.
*
* @remark The `HDC` associated with the window can be queried with the
* [GetDC](https://docs.microsoft.com/en-us/windows/win32/api/winuser/nf-winuser-getdc)
* function.
* @code
* HDC dc = GetDC(glfwGetWin32Window(window));
* @endcode
* This DC is private and does not need to be released.
*
* @thread_safety This function may be called from any thread. Access is not
* synchronized.
*
* @since Added in version 3.0.
*
* @ingroup native
*/
GLFWAPI HGLRC glfwGetWGLContext(GLFWwindow *window);
#endif
#if defined(GLFW_EXPOSE_NATIVE_COCOA)
/*! @brief Returns the `CGDirectDisplayID` of the specified monitor.
*
* @return The `CGDirectDisplayID` of the specified monitor, or
* `kCGNullDirectDisplay` if an [error](@ref error_handling) occurred.
*
* @errors Possible errors include @ref GLFW_NOT_INITIALIZED and @ref
* GLFW_PLATFORM_UNAVAILABLE.
*
* @thread_safety This function may be called from any thread. Access is not
* synchronized.
*
* @since Added in version 3.1.
*
* @ingroup native
*/
GLFWAPI CGDirectDisplayID glfwGetCocoaMonitor(GLFWmonitor *monitor);
/*! @brief Returns the `NSWindow` of the specified window.
*
* @return The `NSWindow` of the specified window, or `nil` if an
* [error](@ref error_handling) occurred.
*
* @errors Possible errors include @ref GLFW_NOT_INITIALIZED and @ref
* GLFW_PLATFORM_UNAVAILABLE.
*
* @thread_safety This function may be called from any thread. Access is not
* synchronized.
*
* @since Added in version 3.0.
*
* @ingroup native
*/
GLFWAPI id glfwGetCocoaWindow(GLFWwindow *window);
/*! @brief Returns the `NSView` of the specified window.
*
* @return The `NSView` of the specified window, or `nil` if an
* [error](@ref error_handling) occurred.
*
* @errors Possible errors include @ref GLFW_NOT_INITIALIZED and @ref
* GLFW_PLATFORM_UNAVAILABLE.
*
* @thread_safety This function may be called from any thread. Access is not
* synchronized.
*
* @since Added in version 3.4.
*
* @ingroup native
*/
GLFWAPI id glfwGetCocoaView(GLFWwindow *window);
#endif
#if defined(GLFW_EXPOSE_NATIVE_NSGL)
/*! @brief Returns the `NSOpenGLContext` of the specified window.
*
* @return The `NSOpenGLContext` of the specified window, or `nil` if an
* [error](@ref error_handling) occurred.
*
* @errors Possible errors include @ref GLFW_NOT_INITIALIZED, @ref
* GLFW_PLATFORM_UNAVAILABLE and @ref GLFW_NO_WINDOW_CONTEXT.
*
* @thread_safety This function may be called from any thread. Access is not
* synchronized.
*
* @since Added in version 3.0.
*
* @ingroup native
*/
GLFWAPI id glfwGetNSGLContext(GLFWwindow *window);
#endif
#if defined(GLFW_EXPOSE_NATIVE_X11)
/*! @brief Returns the `Display` used by GLFW.
*
* @return The `Display` used by GLFW, or `NULL` if an
* [error](@ref error_handling) occurred.
*
* @errors Possible errors include @ref GLFW_NOT_INITIALIZED and @ref
* GLFW_PLATFORM_UNAVAILABLE.
*
* @thread_safety This function may be called from any thread. Access is not
* synchronized.
*
* @since Added in version 3.0.
*
* @ingroup native
*/
GLFWAPI Display *glfwGetX11Display(void);
/*! @brief Returns the `RRCrtc` of the specified monitor.
*
* @return The `RRCrtc` of the specified monitor, or `None` if an
* [error](@ref error_handling) occurred.
*
* @errors Possible errors include @ref GLFW_NOT_INITIALIZED and @ref
* GLFW_PLATFORM_UNAVAILABLE.
*
* @thread_safety This function may be called from any thread. Access is not
* synchronized.
*
* @since Added in version 3.1.
*
* @ingroup native
*/
GLFWAPI RRCrtc glfwGetX11Adapter(GLFWmonitor *monitor);
/*! @brief Returns the `RROutput` of the specified monitor.
*
* @return The `RROutput` of the specified monitor, or `None` if an
* [error](@ref error_handling) occurred.
*
* @errors Possible errors include @ref GLFW_NOT_INITIALIZED and @ref
* GLFW_PLATFORM_UNAVAILABLE.
*
* @thread_safety This function may be called from any thread. Access is not
* synchronized.
*
* @since Added in version 3.1.
*
* @ingroup native
*/
GLFWAPI RROutput glfwGetX11Monitor(GLFWmonitor *monitor);
/*! @brief Returns the `Window` of the specified window.
*
* @return The `Window` of the specified window, or `None` if an
* [error](@ref error_handling) occurred.
*
* @errors Possible errors include @ref GLFW_NOT_INITIALIZED and @ref
* GLFW_PLATFORM_UNAVAILABLE.
*
* @thread_safety This function may be called from any thread. Access is not
* synchronized.
*
* @since Added in version 3.0.
*
* @ingroup native
*/
GLFWAPI Window glfwGetX11Window(GLFWwindow *window);
/*! @brief Sets the current primary selection to the specified string.
*
* @param[in] string A UTF-8 encoded string.
*
* @errors Possible errors include @ref GLFW_NOT_INITIALIZED, @ref
* GLFW_PLATFORM_UNAVAILABLE and @ref GLFW_PLATFORM_ERROR.
*
* @pointer_lifetime The specified string is copied before this function
* returns.
*
* @thread_safety This function must only be called from the main thread.
*
* @sa @ref clipboard
* @sa glfwGetX11SelectionString
* @sa glfwSetClipboardString
*
* @since Added in version 3.3.
*
* @ingroup native
*/
GLFWAPI void glfwSetX11SelectionString(const char *string);
/*! @brief Returns the contents of the current primary selection as a string.
*
* If the selection is empty or if its contents cannot be converted, `NULL`
* is returned and a @ref GLFW_FORMAT_UNAVAILABLE error is generated.
*
* @return The contents of the selection as a UTF-8 encoded string, or `NULL`
* if an [error](@ref error_handling) occurred.
*
* @errors Possible errors include @ref GLFW_NOT_INITIALIZED, @ref
* GLFW_PLATFORM_UNAVAILABLE and @ref GLFW_PLATFORM_ERROR.
*
* @pointer_lifetime The returned string is allocated and freed by GLFW. You
* should not free it yourself. It is valid until the next call to @ref
* glfwGetX11SelectionString or @ref glfwSetX11SelectionString, or until the
* library is terminated.
*
* @thread_safety This function must only be called from the main thread.
*
* @sa @ref clipboard
* @sa glfwSetX11SelectionString
* @sa glfwGetClipboardString
*
* @since Added in version 3.3.
*
* @ingroup native
*/
GLFWAPI const char *glfwGetX11SelectionString(void);
#endif
#if defined(GLFW_EXPOSE_NATIVE_GLX)
/*! @brief Returns the `GLXContext` of the specified window.
*
* @return The `GLXContext` of the specified window, or `NULL` if an
* [error](@ref error_handling) occurred.
*
* @errors Possible errors include @ref GLFW_NOT_INITIALIZED, @ref
* GLFW_NO_WINDOW_CONTEXT and @ref GLFW_PLATFORM_UNAVAILABLE.
*
* @thread_safety This function may be called from any thread. Access is not
* synchronized.
*
* @since Added in version 3.0.
*
* @ingroup native
*/
GLFWAPI GLXContext glfwGetGLXContext(GLFWwindow *window);
/*! @brief Returns the `GLXWindow` of the specified window.
*
* @return The `GLXWindow` of the specified window, or `None` if an
* [error](@ref error_handling) occurred.
*
* @errors Possible errors include @ref GLFW_NOT_INITIALIZED, @ref
* GLFW_NO_WINDOW_CONTEXT and @ref GLFW_PLATFORM_UNAVAILABLE.
*
* @thread_safety This function may be called from any thread. Access is not
* synchronized.
*
* @since Added in version 3.2.
*
* @ingroup native
*/
GLFWAPI GLXWindow glfwGetGLXWindow(GLFWwindow *window);
#endif
#if defined(GLFW_EXPOSE_NATIVE_WAYLAND)
/*! @brief Returns the `struct wl_display*` used by GLFW.
*
* @return The `struct wl_display*` used by GLFW, or `NULL` if an
* [error](@ref error_handling) occurred.
*
* @errors Possible errors include @ref GLFW_NOT_INITIALIZED and @ref
* GLFW_PLATFORM_UNAVAILABLE.
*
* @thread_safety This function may be called from any thread. Access is not
* synchronized.
*
* @since Added in version 3.2.
*
* @ingroup native
*/
GLFWAPI struct wl_display *glfwGetWaylandDisplay(void);
/*! @brief Returns the `struct wl_output*` of the specified monitor.
*
* @return The `struct wl_output*` of the specified monitor, or `NULL` if an
* [error](@ref error_handling) occurred.
*
* @errors Possible errors include @ref GLFW_NOT_INITIALIZED and @ref
* GLFW_PLATFORM_UNAVAILABLE.
*
* @thread_safety This function may be called from any thread. Access is not
* synchronized.
*
* @since Added in version 3.2.
*
* @ingroup native
*/
GLFWAPI struct wl_output *glfwGetWaylandMonitor(GLFWmonitor *monitor);
/*! @brief Returns the main `struct wl_surface*` of the specified window.
*
* @return The main `struct wl_surface*` of the specified window, or `NULL` if
* an [error](@ref error_handling) occurred.
*
* @errors Possible errors include @ref GLFW_NOT_INITIALIZED and @ref
* GLFW_PLATFORM_UNAVAILABLE.
*
* @thread_safety This function may be called from any thread. Access is not
* synchronized.
*
* @since Added in version 3.2.
*
* @ingroup native
*/
GLFWAPI struct wl_surface *glfwGetWaylandWindow(GLFWwindow *window);
#endif
#if defined(GLFW_EXPOSE_NATIVE_EGL)
/*! @brief Returns the `EGLDisplay` used by GLFW.
*
* @return The `EGLDisplay` used by GLFW, or `EGL_NO_DISPLAY` if an
* [error](@ref error_handling) occurred.
*
* @errors Possible errors include @ref GLFW_NOT_INITIALIZED.
*
* @remark Because EGL is initialized on demand, this function will return
* `EGL_NO_DISPLAY` until the first context has been created via EGL.
*
* @thread_safety This function may be called from any thread. Access is not
* synchronized.
*
* @since Added in version 3.0.
*
* @ingroup native
*/
GLFWAPI EGLDisplay glfwGetEGLDisplay(void);
/*! @brief Returns the `EGLContext` of the specified window.
*
* @return The `EGLContext` of the specified window, or `EGL_NO_CONTEXT` if an
* [error](@ref error_handling) occurred.
*
* @errors Possible errors include @ref GLFW_NOT_INITIALIZED and @ref
* GLFW_NO_WINDOW_CONTEXT.
*
* @thread_safety This function may be called from any thread. Access is not
* synchronized.
*
* @since Added in version 3.0.
*
* @ingroup native
*/
GLFWAPI EGLContext glfwGetEGLContext(GLFWwindow *window);
/*! @brief Returns the `EGLSurface` of the specified window.
*
* @return The `EGLSurface` of the specified window, or `EGL_NO_SURFACE` if an
* [error](@ref error_handling) occurred.
*
* @errors Possible errors include @ref GLFW_NOT_INITIALIZED and @ref
* GLFW_NO_WINDOW_CONTEXT.
*
* @thread_safety This function may be called from any thread. Access is not
* synchronized.
*
* @since Added in version 3.0.
*
* @ingroup native
*/
GLFWAPI EGLSurface glfwGetEGLSurface(GLFWwindow *window);
#endif
#if defined(GLFW_EXPOSE_NATIVE_OSMESA)
/*! @brief Retrieves the color buffer associated with the specified window.
*
* @param[in] window The window whose color buffer to retrieve.
* @param[out] width Where to store the width of the color buffer, or `NULL`.
* @param[out] height Where to store the height of the color buffer, or `NULL`.
* @param[out] format Where to store the OSMesa pixel format of the color
* buffer, or `NULL`.
* @param[out] buffer Where to store the address of the color buffer, or
* `NULL`.
* @return `GLFW_TRUE` if successful, or `GLFW_FALSE` if an
* [error](@ref error_handling) occurred.
*
* @errors Possible errors include @ref GLFW_NOT_INITIALIZED and @ref
* GLFW_NO_WINDOW_CONTEXT.
*
* @thread_safety This function may be called from any thread. Access is not
* synchronized.
*
* @since Added in version 3.3.
*
* @ingroup native
*/
GLFWAPI int glfwGetOSMesaColorBuffer(GLFWwindow *window, int *width, int *height, int *format,
void **buffer);
/*! @brief Retrieves the depth buffer associated with the specified window.
*
* @param[in] window The window whose depth buffer to retrieve.
* @param[out] width Where to store the width of the depth buffer, or `NULL`.
* @param[out] height Where to store the height of the depth buffer, or `NULL`.
* @param[out] bytesPerValue Where to store the number of bytes per depth
* buffer element, or `NULL`.
* @param[out] buffer Where to store the address of the depth buffer, or
* `NULL`.
* @return `GLFW_TRUE` if successful, or `GLFW_FALSE` if an
* [error](@ref error_handling) occurred.
*
* @errors Possible errors include @ref GLFW_NOT_INITIALIZED and @ref
* GLFW_NO_WINDOW_CONTEXT.
*
* @thread_safety This function may be called from any thread. Access is not
* synchronized.
*
* @since Added in version 3.3.
*
* @ingroup native
*/
GLFWAPI int glfwGetOSMesaDepthBuffer(GLFWwindow *window, int *width, int *height,
int *bytesPerValue, void **buffer);
/*! @brief Returns the `OSMesaContext` of the specified window.
*
* @return The `OSMesaContext` of the specified window, or `NULL` if an
* [error](@ref error_handling) occurred.
*
* @errors Possible errors include @ref GLFW_NOT_INITIALIZED and @ref
* GLFW_NO_WINDOW_CONTEXT.
*
* @thread_safety This function may be called from any thread. Access is not
* synchronized.
*
* @since Added in version 3.3.
*
* @ingroup native
*/
GLFWAPI OSMesaContext glfwGetOSMesaContext(GLFWwindow *window);
#endif
#ifdef __cplusplus
}
#endif
#endif /* _glfw3_native_h_ */
@@ -0,0 +1,3 @@
cmake_minimum_required(VERSION 3.16)
sn_register_external_package(${STGSHARP_NATIVE_EXTERNAL_TARGET} stbi ${CMAKE_CURRENT_LIST_DIR})
File diff suppressed because it is too large Load Diff
+9
View File
@@ -0,0 +1,9 @@
# StgSharp c file naming regulation
| Name Header | Description |
|:-------:|:--------|
|sn-|Stgsharp Native lib self operation|
|k-|Matrix kernel operation|
|str-|SIMD boosted stream operation|
|load-|Context loading operation|
|##OTHER##-|File from another open source lib with name ##OTHER##|
@@ -0,0 +1,35 @@
#ifndef SNative
#define SNative
#ifdef __cplusplus
#error "This header is not meant to be included in C++ code. Nitload.Native is a C library and should be used with C linkage."
#endif
#include "sn_internal.h"
#include "sn_target.h"
#include <immintrin.h>
#include <assert.h>
#include <glfw_function.h>
#define byte char
#define cs_char uint16_t
#define sn_null_assert(ptr) assert(ptr != NULL)
#ifndef SN_
#endif
SN_API void SN_DECL load_glfw_functions(GLFWFunctionTable *table);
SN_API void SN_DECL load_intrinsic_function(void *intrinsic_context, uint64_t id);
SN_API SIMDID SN_DECL sn_get_simd_level_global();
/* Per-core variant: always re-executes CPUID, never caches.
* Must be called on the target core (after affinity is set). */
SN_API SIMDID SN_DECL sn_get_simd_level_local();
SN_API SIMDID SN_DECL sn_get_unite_simd(SIMDID left, SIMDID right);
SN_API int SN_DECL sn_compare_simd(SIMDID left, SIMDID right);
SN_API void SN_DECL sn_set_thread_affinity(int core_id, int numa_id);
SN_API void* SN_DECL sn_numa_alloc(size_t size, int numa_id);
SN_API void SN_DECL sn_numa_free(void *ptr, size_t size, int numa_id);
SN_API int SN_DECL sn_get_numa_each(int numa_id, uint64_t *internal, uint32_t *global);
SN_API int SN_DECL sn_get_numa_overall(int *out_numa_count, int *count_of_each_numa);
#endif
+150
View File
@@ -0,0 +1,150 @@
#ifndef GLFW_FUNCTION_TABLE_H
#define GLFW_FUNCTION_TABLE_H
#ifndef GLFW_INCLUDE_NONE
#define GLFW_INCLUDE_NONE
#endif
#include "../lib/glfw/glfw3.h"
#ifdef __cplusplus
extern "C" {
#endif
typedef struct GLFWFunctionTable {
GLFWcursor *(*glfwCreateCursor)(const GLFWimage *image, int xhot, int yhot);
GLFWcursor *(*glfwCreateStandardCursor)(int shape);
GLFWwindow *(*glfwCreateWindow)(int width, int height, const char *title,
GLFWmonitor *monitor, GLFWwindow *share);
void (*glfwDefaultWindowHints)(void);
void (*glfwDestroyCursor)(GLFWcursor *cursor);
void (*glfwDestroyWindow)(GLFWwindow *window);
int (*glfwExtensionSupported)(const char *extension);
void (*glfwFocusWindow)(GLFWwindow *window);
const char *(*glfwGetClipboardString)(GLFWwindow *window);
GLFWwindow *(*glfwGetCurrentContext)(void);
void (*glfwGetCursorPos)(GLFWwindow *window, double *xpos, double *ypos);
int (*glfwGetError)(const char **description);
void (*glfwGetFrameBufferSize)(GLFWwindow *window, int *width, int *height);
const char *(*glfwGetGamepadName)(int jid);
int (*glfwGetGamepadState)(int jid, GLFWgamepadstate *state);
const GLFWgammaramp *(*glfwGetGammaRamp)(GLFWmonitor *monitor);
int (*glfwGetInputMode)(GLFWwindow *window, int mode);
const float *(*glfwGetJoystickAxes)(int jid, int *count);
const unsigned char *(*glfwGetJoystickButtons)(int jid, int *count);
const char *(*glfwGetJoystickGUID)(int jid);
const unsigned char *(*glfwGetJoystickHats)(int jid, int *count);
const char *(*glfwGetJoystickName)(int jid);
void *(*glfwGetJoystickUserPointer)(int jid);
int (*glfwGetKey)(GLFWwindow *window, int key);
const char *(*glfwGetKeyName)(int key, int scancode);
int (*glfwGetKeyScancode)(int key);
void (*glfwGetMonitorContentScale)(GLFWmonitor *monitor, float *xscale, float *yscale);
const char *(*glfwGetMonitorName)(GLFWmonitor *monitor);
void (*glfwGetMonitorPhysicalSize)(GLFWmonitor *monitor, int *widthMM, int *heightMM);
void (*glfwGetMonitorPos)(GLFWmonitor *monitor, int *xpos, int *ypos);
GLFWmonitor **(*glfwGetMonitors)(int *count);
void *(*glfwGetMonitorUserPointer)(GLFWmonitor *monitor);
void (*glfwGetMonitorWorkarea)(GLFWmonitor *monitor, int *xpos, int *ypos, int *width,
int *height);
int (*glfwGetMouseButton)(GLFWwindow *window, int button);
GLFWmonitor *(*glfwGetPrimaryMonitor)(void);
GLFWglproc (*glfwGetProcAddress)(const char *procname);
const char **(*glfwGetRequiredInstanceExtensions)(unsigned int *count);
double (*glfwGetTime)(void);
unsigned long long (*glfwGetTimerFrequency)(void);
unsigned long long (*glfwGetTimerValue)(void);
void (*glfwGetVersion)(int *major, int *minor, int *rev);
const char *(*glfwGetVersionString)(void);
const GLFWvidmode *(*glfwGetVideoMode)(GLFWmonitor *monitor);
const GLFWvidmode *(*glfwGetVideoModes)(GLFWmonitor *monitor, int *count);
int (*glfwGetWindowAttrib)(GLFWwindow *window, int attrib);
void (*glfwGetWindowContentScale)(GLFWwindow *window, float *xscale, float *yscale);
void (*glfwGetWindowFrameSize)(GLFWwindow *window, int *left, int *top, int *right,
int *bottom);
GLFWmonitor *(*glfwGetWindowMonitor)(GLFWwindow *window);
float (*glfwGetWindowOpacity)(GLFWwindow *window);
void (*glfwGetWindowPos)(GLFWwindow *window, int *xpos, int *ypos);
void (*glfwGetWindowSize)(GLFWwindow *window, int *width, int *height);
void *(*glfwGetWindowUserPointer)(GLFWwindow *window);
void (*glfwHideWindow)(GLFWwindow *window);
void (*glfwIconifyWindow)(GLFWwindow *window);
int (*glfwInit)(void);
void (*glfwInitHint)(int hint, int value);
int (*glfwJoystickIsGamePad)(int jid);
int (*glfwJoystickPresent)(int jid);
void (*glfwMakeContextCurrent)(GLFWwindow *window);
void (*glfwMaximizeWindow)(GLFWwindow *window);
void (*glfwPollEvents)(void);
void (*glfwPostEmptyEvent)(void);
int (*glfwRawMouseMotionSupported)(void);
void (*glfwRequestWindowAttention)(GLFWwindow *window);
void (*glfwRestoreWindow)(GLFWwindow *window);
GLFWcharfun (*glfwSetCharCallback)(GLFWwindow *window, GLFWcharfun callback);
GLFWcharmodsfun (*glfwSetCharModsCallback)(GLFWwindow *window, GLFWcharmodsfun callback);
void (*glfwSetClipboardString)(GLFWwindow *window, const char *str);
void (*glfwSetCursor)(GLFWwindow *window, GLFWcursor *cursor);
GLFWcursorenterfun (*glfwSetCursorEnterCallback)(GLFWwindow *window,
GLFWcursorenterfun callback);
void (*glfwSetCursorPos)(GLFWwindow *window, double xpos, double ypos);
GLFWcursorposfun (*glfwSetCursorPosCallback)(GLFWwindow *window, GLFWcursorposfun callback);
GLFWdropfun (*glfwSetDropCallback)(GLFWwindow *window, GLFWdropfun callback);
GLFWerrorfun (*glfwSetErrorCallback)(GLFWerrorfun callback);
GLFWframebuffersizefun (*glfwSetFramebufferSizeCallback)(GLFWwindow *window,
GLFWframebuffersizefun callback);
void (*glfwSetGamma)(GLFWmonitor *monitor, float gamma);
void (*glfwSetGammaRamp)(GLFWmonitor *monitor, const GLFWgammaramp *ramp);
void (*glfwSetInputMode)(GLFWwindow *window, int mode, int value);
GLFWjoystickfun (*glfwSetJoystickCallback)(GLFWjoystickfun callback);
void (*glfwSetJoystickUserPointer)(int jid, void *pointer);
GLFWkeyfun (*glfwSetKeyCallback)(GLFWwindow *window, GLFWkeyfun callback);
GLFWmonitorfun (*glfwSetMonitorCallback)(GLFWmonitorfun callback);
void (*glfwSetMonitorUserPointer)(GLFWmonitor *monitor, void *pointer);
GLFWmousebuttonfun (*glfwSetMouseButtonCallback)(GLFWwindow *window,
GLFWmousebuttonfun callback);
GLFWscrollfun (*glfwSetScrollCallback)(GLFWwindow *window, GLFWscrollfun callback);
void (*glfwSetTime)(double time);
void (*glfwSetWindowAspectRatio)(GLFWwindow *window, int numer, int denom);
void (*glfwSetWindowAttrib)(GLFWwindow *window, int attrib, int value);
GLFWwindowclosefun (*glfwSetWindowCloseCallback)(GLFWwindow *window,
GLFWwindowclosefun callback);
GLFWwindowcontentscalefun (*glfwSetWindowContentScaleCallback)(
GLFWwindow *window, GLFWwindowcontentscalefun callback);
GLFWwindowfocusfun (*glfwSetWindowFocusCallback)(GLFWwindow *window,
GLFWwindowfocusfun callback);
void (*glfwSetWindowIcon)(GLFWwindow *window, int count, const GLFWimage *images);
GLFWwindowiconifyfun (*glfwSetWindowIconifyCallback)(GLFWwindow *window,
GLFWwindowiconifyfun callback);
GLFWwindowmaximizefun (*glfwSetWindowMaximizeCallback)(GLFWwindow *window,
GLFWwindowmaximizefun callback);
void (*glfwSetWindowMonitor)(GLFWwindow *window, GLFWmonitor *monitor, int xpos, int ypos,
int width, int height, int refreshRate);
void (*glfwSetWindowOpacity)(GLFWwindow *window, float opacity);
void (*glfwSetWindowPos)(GLFWwindow *window, int xpos, int ypos);
GLFWwindowposfun (*glfwSetWindowPosCallback)(GLFWwindow *window, GLFWwindowposfun callback);
GLFWwindowrefreshfun (*glfwSetWindowRefreshCallback)(GLFWwindow *window,
GLFWwindowrefreshfun callback);
void (*glfwSetWindowShouldClose)(GLFWwindow *window, int value);
void (*glfwSetWindowSize)(GLFWwindow *window, int width, int height);
GLFWwindowsizefun (*glfwSetWindowSizeCallback)(GLFWwindow *window,
GLFWwindowsizefun callback);
void (*glfwSetWindowSizeLimits)(GLFWwindow *window, int minwidth, int minheight,
int maxwidth, int maxheight);
void (*glfwSetWindowTitle)(GLFWwindow *window, const char *title);
void (*glfwSetWindowUserPointer)(GLFWwindow *window, void *pointer);
void (*glfwShowWindow)(GLFWwindow *window);
void (*glfwSwapBuffers)(GLFWwindow *window);
void (*glfwSwapInterval)(int interval);
void (*glfwTerminate)(void);
int (*glfwUpdateGamePadMappings)(const char *str);
int (*glfwVulkanSupported)(void);
void (*glfwWaitEvents)(void);
void (*glfwWaitEventsTimeout)(double timeout);
void (*glfwWindowHint)(int hint, int value);
void (*glfwWindowHintString)(int hint, const char *value);
int (*glfwWindowShouldClose)(GLFWwindow *window);
} GLFWFunctionTable;
#ifdef __cplusplus
}
#endif
#endif
+45
View File
@@ -0,0 +1,45 @@
#ifndef SN_INTERNAL
#define SN_INTERNAL
#include <xmmintrin.h>
#define UNSAFE_AS(type, expr) *(type *)&(expr)
#ifdef _MSC_VER
#define _CRT_SECURE_NO_WARNINGS 1
#define WIN32_LEAN_AND_MEAN
#define SN_DECL __cdecl
#define SN_MAT_DECL __cdecl __forceinline
#define SN_API __declspec(dllexport)
#define FORCEINLINE __forceinline
#define NOINLINE __declspec(noinline)
#include <windows.h>
#elif defined(__clang__) || defined(__GNUC__)
#define SN_DECL __attribute__((cdecl))
#define SN_MAT_DECL __attribute__((cdecl)) __attribute__((always_inline))
#define SN_API __attribute__((dllexport))
#define FORCEINLINE __attribute__((always_inline))
#define NOINLINE __attribute__((noinline))
#else
#define SN_API
#define SN_DECL
#define SN_MAT_DECL
#define FORCEINLINE
#define NOINLINE
#endif
#if defined(_MSC_VER)
#define SN_ALIGNAS(n) __declspec(align(n))
#else
#define SN_ALIGNAS(n) __attribute__((aligned(n)))
#endif
#define SN_APIDEF
#define PRIVATE static
#define INTERNAL extern
#define u64 uint64_t
#endif //
@@ -0,0 +1,34 @@
#ifndef SN_INTRINSIC_CONTEXT_BASE
#define SN_INTRINSIC_CONTEXT_BASE
// -----------------------------------------------------------------------------
// SIMDID layout helpers (see SIMDID.x86.md)
// bits [3..0] : Root ISA
// bits [7..4] : Manufacture (ISA-specific)
// bits [15..8] : MainLevel (monotonic capability)
// -----------------------------------------------------------------------------
#define SIMDID_SHIFT_ROOT 0
#define SIMDID_SHIFT_MANU 4
#define SIMDID_SHIFT_MAIN 8
#ifndef SIMDID_PACK_ROOT
#define SIMDID_PACK_ROOT(root) (((uint64_t)(root) & 0xFULL) << SIMDID_SHIFT_ROOT)
#define SIMDID_PACK_MANU(manu) (((uint64_t)(manu) & 0xFULL) << SIMDID_SHIFT_MANU)
#define SIMDID_PACK_MAIN(main) (((uint64_t)(main) & 0xFFULL) << SIMDID_SHIFT_MAIN)
#endif
// Root ISA (low nibble)
#define SIMDID_ROOT_UNKNOWN 0x0 // cannot find root isa, error branch
#define SIMDID_ROOT_X86_64 0x1 // support
#define SIMDID_ROOT_AARCH64 0x2 // schedualed
#define SIMDID_ROOT_PPC64LE 0x3 // willing, but no plan yet
#define SIMDID_ROOT_RISCV64 0x4 // willing, but no plan yet
#define SIMDID_ROOT_LOONGARCH 0x5 // willing, but no plan yet
#define SIMDID_MASK_ROOT 0xFULL
#define SIMDID_MASK_MANU (0xFULL << SIMDID_SHIFT_MANU)
#define SIMDID_MASK_MAIN (0xFFULL << SIMDID_SHIFT_MAIN)
#endif
@@ -0,0 +1,294 @@
#ifndef SN_INTRINSIC_CONTEXT_X86_H
#define SN_INTRINSIC_CONTEXT_X86_H
#include "sn_target.h"
#if SN_IS_ARCH(SN_ARCH_X86_64)
#include "sn_intrinsic.h"
#include "sn_intrinsic.std.h"
#include "sn_intrinsic.context.base.h"
#ifndef SN_X86_TARGET_ATTR_sse
#define SN_STD_TARGET_ATTR_IMPL(T_arch, T_feature) SN_X86_TARGET_ATTR_##T_arch##T_feature
#define SN_X86_TARGET_ATTR_sse SN_CLANG_TARGET_ATTR("sse4.2")
#define SN_X86_TARGET_ATTR_avx SN_CLANG_TARGET_ATTR("avx")
#define SN_X86_TARGET_ATTR_avx_fma SN_CLANG_TARGET_ATTR("avx,avx2,fma")
#define SN_X86_TARGET_ATTR_512 SN_CLANG_TARGET_ATTR("avx512f,avx512vl")
#endif
// -----------------------------------------------------------------------------
// SIMDID layout helpers (see SIMDID.x86.md)
// bits [3..0] : Root ISA
// bits [7..4] : Manufacture (ISA-specific)
// bits [15..8] : MainLevel (monotonic capability)
// bits [31..16]: AVX512 Feature
// bits [39..32]: AVX Feature
// bits [47..40]: AVX10 Feature
// bits [55..48]: uArchHi / Policy (optional)
// bits [63..56]: Reserved for future tail extension
// -----------------------------------------------------------------------------
#define SIMDID_SHIFT_AVX512 16
#define SIMDID_SHIFT_AVX 32
#define SIMDID_SHIFT_AVX10 40
#define SIMDID_SHIFT_UARCH 48
#define SIMDID_MASK_AVX512 (0xFFFFULL << SIMDID_SHIFT_AVX512)
#define SIMDID_MASK_AVX (0xFFULL << SIMDID_SHIFT_AVX)
#define SIMDID_MASK_AVX10 (0xFFULL << SIMDID_SHIFT_AVX10)
#define SIMDID_MASK_UARCH (0xFFULL << SIMDID_SHIFT_UARCH)
// Manufacture (x86-64 mapping; other ISAs may reinterpret)
#define SIMDID_MANU_UNKNOWN 0x0
#define SIMDID_MANU_INTEL 0x1
#define SIMDID_MANU_AMD 0x2
#define SIMDID_MANU_VIA 0x3
#define SIMDID_MANU_HYGON 0x4 /* 海光: Zen1-derived, separate from AMD */
// MainLevel (ordinal, shared bits 8..15; not bit-flags)
#define SIMDID_MAIN_LVL_NONE 0x00u
#define SIMDID_MAIN_LVL_SSE 0x01u /* SSE1..SSE4.2 */
#define SIMDID_MAIN_LVL_AVX2 0x02u
#define SIMDID_MAIN_LVL_AVX512 0x03u
#define SIMDID_MAIN_LVL_AVX10 0x04u
#ifndef SIMDID_AVX512_BASE
#define SIMDID_AVX512_VNNI (1U << 11)
#define SIMDID_AVX512_BASE (1U << 4)
#define SIMDID_AVX512_IMPL_SHIFT 6
#define SIMDID_AVX512_IMPL_MASK (0x3U << SIMDID_AVX512_IMPL_SHIFT)
#define SIMDID_AVX512_IMPL_UNKNOWN (0x0U << SIMDID_AVX512_IMPL_SHIFT)
#define SIMDID_AVX512_IMPL_DUAL (0x1U << SIMDID_AVX512_IMPL_SHIFT)
#define SIMDID_AVX512_IMPL_NATIVE (0x2U << SIMDID_AVX512_IMPL_SHIFT)
#endif
#ifndef SIMDID_AVX_BASE
#define SIMDID_AVX (1U << 0) /* relative bit 0 within [39..32] */
#define SIMDID_AVX2 (1U << 1)
#define SIMDID_AVX_FMA (1U << 2)
#define SIMDID_AVX_F16C (1U << 3)
#define SIMDID_AVX_DUAL \
(1U << 4) /* glued/split-lane YMM (two 128-bit units; Zen1, Hygon, VIA/Zhaoxin) */
#endif
#ifndef SIMDID_AVX10_VERSION_MASK
#define SIMDID_AVX10_VERSION_MASK 0x7FU /* CPUID.24H.0:EBX[7:0], saturated to 7 bits */
#define SIMDID_AVX10_VNNI_INT (1U << 7) /* CPUID.24H.1:ECX[2] */
#endif
/* uArchHi ([55..48]) bitmask — core topology and SIMD-width execution policy */
#ifndef SIMDID_UARCH_HYBRID_P
/* Topology 2-bit field (bits [1..0] of uArchHi byte):
* 00 = unknown (undetected hybrid type or pre-CPUID.1A hardware)
* 01 = E-core (efficiency/small core in a hybrid CPU)
* 10 = P-core (performance/big core in a hybrid CPU)
* 11 = unified (confirmed non-hybrid; numerically strongest)
* Higher numeric value == stronger / higher-capability topology. */
#define SIMDID_UARCH_HYBRID_E (1U << 0) /* E-core (efficiency/small) in a hybrid topology */
#define SIMDID_UARCH_HYBRID_P (1U << 1) /* P-core (performance/big) in a hybrid topology */
#define SIMDID_UARCH_SIMD_DUAL (1U << 2) /* SIMD is glued/dual (summary: AVX_DUAL or AVX512_IMPL=DUAL) */
#endif
// Pack helpers (define locally to avoid missing macros when included standalone)
#ifndef SIMDID_PACK_X86
#define SIMDID_PACK_X86(root, manu, main) \
(SIMDID_PACK_ROOT(root) | SIMDID_PACK_MANU(manu) | SIMDID_PACK_MAIN(main))
#define SIMDID_PACK_AVX512(avx512_bits) \
(((uint64_t)(avx512_bits) & 0xFFFFULL) << SIMDID_SHIFT_AVX512)
#define SIMDID_PACK_AVX(avx_bits) (((uint64_t)(avx_bits) & 0xFFULL) << SIMDID_SHIFT_AVX)
#define SIMDID_PACK_AVX10(avx10_bits) (((uint64_t)(avx10_bits) & 0xFFULL) << SIMDID_SHIFT_AVX10)
#define SIMDID_PACK_UARCH(uarch_hi) (((uint64_t)(uarch_hi) & 0xFFULL) << SIMDID_SHIFT_UARCH)
#endif
/* Pre-shifted helpers for convenience when constructing SIMDID */
#define SIMDID_MAIN_ENC_NONE SIMDID_PACK_MAIN(SIMDID_MAIN_LVL_NONE)
#define SIMDID_MAIN_ENC_SSE SIMDID_PACK_MAIN(SIMDID_MAIN_LVL_SSE)
#define SIMDID_MAIN_ENC_AVX2 SIMDID_PACK_MAIN(SIMDID_MAIN_LVL_AVX2)
#define SIMDID_MAIN_ENC_AVX512 SIMDID_PACK_MAIN(SIMDID_MAIN_LVL_AVX512)
#define SIMDID_MAIN_ENC_AVX10 SIMDID_PACK_MAIN(SIMDID_MAIN_LVL_AVX10)
#pragma region matix function
INTERNAL void f32_normalize(VEC_SEGMENT(float) * source, VEC_SEGMENT(float) * target);
SN_MK_PROC_DECL_STD(float, sse, , k_quality);
SN_MK_PROC_DECL_STD(float, sse, , add);
SN_MK_PROC_DECL_STD(float, avx, , add);
SN_MK_PROC_DECL_STD(float, 512, , add);
SN_MK_PROC_DECL_STD(double, sse, , add);
SN_MK_PROC_DECL_STD(double, avx, , add);
SN_MK_PROC_DECL_STD(double, 512, , add);
SN_MK_PROC_DECL_STD(float, sse, , sub);
SN_MK_PROC_DECL_STD(float, avx, , sub);
SN_MK_PROC_DECL_STD(float, 512, , sub);
SN_MK_PROC_DECL_STD(double, sse, , sub);
SN_MK_PROC_DECL_STD(double, avx, , sub);
SN_MK_PROC_DECL_STD(double, 512, , sub);
SN_MK_PROC_DECL_STD(float, sse, , transpose);
SN_MK_PROC_DECL_STD(float, avx, , transpose);
SN_MK_PROC_DECL_STD(float, 512, , transpose);
SN_MK_PROC_DECL_STD(float, sse, , scalar_mul);
SN_MK_PROC_DECL_STD(float, avx, , scalar_mul);
SN_MK_PROC_DECL_STD(float, 512, , scalar_mul);
SN_MK_PROC_DECL_STD(double, sse, , scalar_mul);
SN_MK_PROC_DECL_STD(double, avx, , scalar_mul);
SN_MK_PROC_DECL_STD(double, 512, , scalar_mul);
SN_MK_PROC_DECL_STD(float, sse, , fill);
SN_MK_PROC_DECL_STD(float, avx, , fill);
SN_MK_PROC_DECL_STD(float, 512, , fill);
SN_MK_PROC_DECL_STD(double, sse, , fill);
SN_MK_PROC_DECL_STD(double, avx, , fill);
SN_MK_PROC_DECL_STD(double, 512, , fill);
SN_MK_PROC_DECL_STD(float, sse, , quality);
/*
* pack left matrix data to L4 buffer
*
* CONVENSIONS:
*
* buffer_1 is pointer to original colum,
* offset_1 is the offset to the original column,
* count_1 is the count can be packed
*
* C# shceduler is responsible for calculating the offset and count based on the actual buffer size and matrix dimension,
* and make sure that count_1 will not exceeds lenght of the column
*
* scalar pack carries column length and offset data:
* [column length, empty, empty, empty]
*/
SN_MK_PROC_DECL_STD(float, sse, , fma_pack_left);
SN_MK_PROC_DECL_STD(float, avx, , fma_pack_left);
SN_MK_PROC_DECL_STD(float, 512, , fma_pack_left);
/*
* pack right matrix data to L4 buffer
*
* CONVENSIONS:
*
* buffer_1 is pointer to original row,
* offset_1 is the offset to the original row,
* count_1 is the count can be packed
*
* C# shceduler is responsible for calculating the offset and count based on the actual buffer size and matrix dimension,
* and make sure that count_1 will not exceeds lenght of the row
*
* scalar pack carries column length and offset data:
* [column length, empty, empty, empty]
*/
SN_MK_PROC_DECL_STD(float, sse, , fma_pack_right);
SN_MK_PROC_DECL_STD(float, avx, , fma_pack_right);
SN_MK_PROC_DECL_STD(float, 512, , fma_pack_right);
SN_MK_PROC_DECL_STD(float, sse, , fma);
SN_MK_PROC_DECL_STD(float, avx, , fma);
SN_MK_PROC_DECL_STD(float, avx, _fma, fma);
SN_MK_PROC_DECL_STD(float, 512, , fma);
SN_MK_PROC_DECL_STD(float, sse, , fma_unpack);
SN_MK_PROC_DECL_STD(float, avx, , fma_unpack);
SN_MK_PROC_DECL_STD(float, 512, , fma_unpack);
SN_MK_PROC_DECL_STD(double, sse, , fma_pack_left);
SN_MK_PROC_DECL_STD(double, avx, , fma_pack_left);
SN_MK_PROC_DECL_STD(double, 512, , fma_pack_left);
SN_MK_PROC_DECL_STD(double, sse, , fma);
SN_MK_PROC_DECL_STD(double, avx, , fma);
SN_MK_PROC_DECL_STD(double, avx, _fma, fma);
SN_MK_PROC_DECL_STD(double, 512, , fma);
SN_MK_PROC_DECL_STD(double, sse, , fma_unpack);
SN_MK_PROC_DECL_STD(double, avx, , fma_unpack);
SN_MK_PROC_DECL_STD(double, 512, , fma_unpack);
SN_SK_PROC_DECL_STD(float, avx, , add);
SN_SK_PROC_DECL_STD(float, 512, , add);
SN_SK_PROC_DECL_STD(double, sse, , add);
SN_SK_PROC_DECL_STD(double, avx, , add);
SN_SK_PROC_DECL_STD(double, 512, , add);
SN_SK_PROC_DECL_STD(float, sse, , sub);
SN_SK_PROC_DECL_STD(float, avx, , sub);
SN_SK_PROC_DECL_STD(float, 512, , sub);
SN_SK_PROC_DECL_STD(double, sse, , sub);
SN_SK_PROC_DECL_STD(double, avx, , sub);
SN_SK_PROC_DECL_STD(double, 512, , sub);
SN_SK_PROC_DECL_STD(float, sse, , transpose);
SN_SK_PROC_DECL_STD(float, avx, , transpose);
SN_SK_PROC_DECL_STD(float, 512, , transpose);
SN_SK_PROC_DECL_STD(float, sse, , scalar_mul);
SN_SK_PROC_DECL_STD(float, avx, , scalar_mul);
SN_SK_PROC_DECL_STD(float, 512, , scalar_mul);
SN_SK_PROC_DECL_STD(double, sse, , scalar_mul);
SN_SK_PROC_DECL_STD(double, avx, , scalar_mul);
SN_SK_PROC_DECL_STD(double, 512, , scalar_mul);
SN_SK_PROC_DECL_STD(float, sse, , fill);
SN_SK_PROC_DECL_STD(float, avx, , fill);
SN_SK_PROC_DECL_STD(float, 512, , fill);
SN_SK_PROC_DECL_STD(double, sse, , fill);
SN_SK_PROC_DECL_STD(double, avx, , fill);
SN_SK_PROC_DECL_STD(double, 512, , fill);
SN_SK_PROC_DECL_STD(float, sse, , quality);
SN_SK_PROC_DECL_STD(float, sse, , fma);
SN_SK_PROC_DECL_STD(float, avx, , fma);
SN_SK_PROC_DECL_STD(float, avx, _fma, fma);
SN_SK_PROC_DECL_STD(float, 512, , fma);
SN_SK_PROC_DECL_STD(double, sse, , fma);
SN_SK_PROC_DECL_STD(double, avx, , fma);
SN_SK_PROC_DECL_STD(double, avx, _fma, fma);
SN_SK_PROC_DECL_STD(double, 512, , fma);
#pragma endregion
#pragma region scaler_simd
INTERNAL uint64_t SN_DECL factorial_simd_sse(int n);
#pragma endregion
#pragma region dot_product
INTERNAL void FORCEINLINE SN_DECL dot_41_sse(MAT_KERNEL(float) const *transpose,
__m128 const *vector, __m128 *ans);
INTERNAL void SN_DECL dot_41_avx(MAT_KERNEL(float) const *transpose, __m128 const *vector,
__m128 *ans);
INTERNAL void SN_DECL dot_42_sse(MAT_KERNEL(float) const *transpose, __m128 const *vector,
__m128 *ans);
INTERNAL void SN_DECL dot_42_avx(MAT_KERNEL(float) const *transpose, __m128 const *vector,
__m128 *ans);
#pragma endregion
#pragma region string
INTERNAL int SN_DECL city_hash_simplify_sse(byte const *str, int length);
INTERNAL int SN_DECL index_pair_sse(short const *stram, uint32_t target, int length);
#pragma endregion
#else
typedef enum intrinsic_level { SIMD0 = 0 } intrinsic_level;
#endif
#endif
+194
View File
@@ -0,0 +1,194 @@
#ifndef SSC_INTRIN
#define SSC_INTRIN
#define SSC_INTRIN_HEAD
#include "StgSharpNative.h"
#include "sn_target.h"
#include <xmmintrin.h>
#include <immintrin.h>
#include <smmintrin.h>
#include "sn_intrinsic.matkernel.h"
#include <stdint.h>
#ifdef _MSC_VER
#include <intrin.h>
#include <isa_availability.h>
#endif
#define LASTTWO(x) (x & 3)
#define LAST(x) (x & 3)
#define get_align_begin(x) ((x + 15) & (~15))
#define SSE_INSERT(source, target, reset3, reset2, reset1, reset0) \
(LASTTWO(source - 1) << 6) | (LASTTWO(target - 1) << 4) | (LAST(reset3) << 3) | \
(LAST(reset2) << 2) | (LAST(reset1) << 1) | LAST(reset0)
#define SSE_INSERT_SIMPLE(source, target) SSE_INSERT(source, target, 0, 0, 0, 0)
#define M256_PERMUTE(s3, s2, s1, s0) \
((LASTTWO(s3) << 6) | (LASTTWO(s2) << 4) | (LASTTWO(s1) << 2) | (LASTTWO(s0)))
#define BIT_CVRT(pack, index, T) *(T *)(pack->data + index)
#if SN_IS_ARCH(x86_64)
#define vec_128 __m128
#define vec_256 __m256
#define vec_512 __m512
#define zero_vec _mm_setzero_ps()
#define zero_vec_256 _mm256_setzero_ps()
#define ALIGN(simdVec) _mm_loadu_ps(&simdVec)
#define ALIGN256(simdVec) _mm256_loadu_ps(&simdVec)
#define ALIGN512(simdVec) _mm512_loadu_ps(&simdVec)
#else
#define zero_vec
#define zero_vec_256
#define ALIGN(simdVec)
#define ALIGN256(simdVec)
#define ALIGN512(simdVec)
#endif
#define SPAN(T) \
struct SPAN_T { \
T *pointer; \
size_t length; \
};
typedef union uint64_u {
uint64_t u64;
uint32_t u32[2];
} uint64_u;
typedef struct mat_kernel mat_kernel;
typedef union M128{
#if SN_IS_ARCH(x86_64)
__m128 m128;
#endif
float f32[4];
double f64[2];
int32_t i32[4];
int64_t i64[2];
}M128;
typedef union M256{
#if SN_IS_ARCH(x86_64)
__m256 m256;
#endif
float f32[8];
double f64[4];
int32_t i32[8];
int64_t i64[4];
}M256;
typedef union M512{
#if SN_IS_ARCH(x86_64)
__m512 m512;
#endif
uint64_t u64[8];
uint32_t u32[16];
float f32[16];
double f64[8];
int32_t i32[16];
int64_t i64[8];
}M512;
typedef struct sn_L4_cache_head {
uint64_t reverse_mask_0;
uint32_t position_mask;
uint32_t size;
uint64_t reverse_mask_1;
uint64_t reverse_mask_2;
} sn_L4_cache_head;
typedef struct sn_L4_handle {
void *ptr;
sn_L4_cache_head* head;
} sn_L4_handle;
typedef struct matrix_task {
sn_L4_handle buffer_0;
sn_L4_handle buffer_1;
sn_L4_handle buffer_2;
sn_L4_handle buffer_3;
union {
struct {
int32_t reverse_0;
int32_t reverse_1;
int32_t offset_0;
int32_t count_0;
int32_t offset_1;
int32_t count_1;
int32_t offset_2;
int32_t count_2;
int32_t offset_3;
int32_t count_3;
int32_t reverse_2;
int32_t reverse_3;
M128 scalar_pack;
};
M512 profile;
};
} matrix_task;
#define MAT_TASK(T) matrix_task
#if defined(__cplusplus)
static_assert(sizeof(matrix_task) == 128, "matrix_parallel_task_package size mismatch");
#elif defined(__STDC_VERSION__)
static_assert(sizeof(matrix_task) == 128, "matrix_parallel_task_package size mismatch");
#endif
typedef void(SN_DECL *UNI_MK_PROC)(matrix_task *task);
typedef void(SN_DECL *UNI_SK_PROC)(mat_kernel const *left, mat_kernel const *right,
mat_kernel *restrict ans, uint64_t scalar);
typedef void(SN_DECL *VECTORNORMALIZEPROC)(VEC_SEGMENT(float) *source, VEC_SEGMENT(float) *target);
typedef void(SN_DECL *DOTPROC)(void *transpose, __m128 *vector, __m128 *ans);
typedef uint64_t(SN_DECL *FACTORIALROC)(int n);
#pragma region string
typedef int(SN_DECL *HASH)(byte const *str, int length);
typedef int(SN_DECL *INDEX_PAIR)(short const *stram, uint32_t target, int length);
#pragma endregion
typedef struct mat_mk_intrinsic {
UNI_MK_PROC proc_array[10];
} mat_mk_intrinsic;
typedef struct mat_sk_intrinsic {
UNI_SK_PROC proc_array[10];
} mat_sk_intrinsic;
// Mirrors C# IntrinsicContext field order so managed/unmanaged layouts stay in sync.
typedef struct sn_intrinsic {
HASH city_hash_simplify;
FACTORIALROC factorial_simd;
VECTORNORMALIZEPROC vec_f32_normalize_3;
mat_mk_intrinsic mat_mk[0x0a];
mat_sk_intrinsic mat_sk[0x0a];
} sn_intrinsic;
typedef enum mat_proc_index {
Add = 0,
Fill = 1,
ScalarMul = 2,
Sub = 3,
Transpose = 4,
ClearPanel = 5,
Fma = 6,
PackFmaLeft = 7,
PackFmaRight = 8,
Pivot = 9,
} mat_proc_index;
typedef enum elment_mask { F32 = 0, F64 = 1, I32 = 2, I64 = 3 } element_mask;
#endif
@@ -0,0 +1,54 @@
#ifndef SN_MATKERNEL
#define SN_MATKERNEL
#include <immintrin.h>
#include <stdint.h>
#include "sn_internal.h"
/* Kernel storage unions for a fixed 4x4 block. */
#define ELEMENTSIZE(T) sizeof(T)
#define XMMCOUNT(T) (sizeof(T))
#define YMMCOUNT(T) (sizeof(T) / 2)
#define ZMMCOUNT(T) (sizeof(T) / 4)
#define SN_MAT_KERNEL_MEMBER(T) \
{ \
__m128 x[XMMCOUNT(T)]; \
__m256 y[YMMCOUNT(T)]; \
__m512 z[ZMMCOUNT(T)]; \
T m[4][4]; \
}
#define SN_MAT_KERNEL_DEFINE(T) \
typedef union SN_MAT_KERNEL_TYPE_##T SN_MAT_KERNEL_MEMBER(T) SN_MAT_KERNEL_TYPE_##T
#define MAT_KERNEL(T) SN_MAT_KERNEL_TYPE(T)
#define SN_MAT_KERNEL_TYPE(T) SN_MAT_KERNEL_TYPE_##T
SN_MAT_KERNEL_DEFINE(float);
SN_MAT_KERNEL_DEFINE(double);
SN_MAT_KERNEL_DEFINE(uint64_t);
#define SN_VEC_SEGMENT_MEMBER(T) \
{ \
__m128 f32_x[XMMCOUNT(T) / 4]; \
__m256 f32_y[YMMCOUNT(T) / 4]; \
__m512 f32_z[ZMMCOUNT(T) / 4]; \
T v[4]; \
}
#define SN_VEC_SEGMENT_DEFINE(T) \
typedef union SN_VEC_SEGMENT_TYPE_##T SN_VEC_SEGMENT_MEMBER(T) SN_VEC_SEGMENT_TYPE_##T
#define VEC_SEGMENT(T) SN_VEC_SEGMENT_TYPE(T)
#define SN_VEC_SEGMENT_TYPE(T) SN_VEC_SEGMENT_TYPE_##T
SN_VEC_SEGMENT_DEFINE(float);
SN_VEC_SEGMENT_DEFINE(double);
/* Global zero kernel declaration (defined in dllmain.c), aligned to 64 bytes */
extern __declspec(align(64)) MAT_KERNEL(u64) SN_ZERO_KERNEL;
#endif /* SN_MATKERNEL */
@@ -0,0 +1,101 @@
#ifndef SN_INTRINSIC_CALLCONV_STD
#define SN_INTRINSIC_CALLCONV_STD
#include "sn_internal.h"
#include "sn_intrinsic.h"
#ifndef static_assert
#define static_assert _Static_assert
#endif
#define SN_STD_TARGET_ATTR(T_arch, T_feature) SN_STD_TARGET_ATTR_IMPL(T_arch, T_feature)
#pragma region mk_proc attributes
#define MK_PROC_NAME_STD(T_type, T_arch, T_feature, T_op) \
T_type##_multi_kernel_proc_##T_op##_##T_arch##_##T_feature
#define GET_MK_PROC_STD(T_type, T_arch, T_feature, T_op) \
((UNI_MK_PROC)MK_PROC_NAME_STD(T_type, T_arch, T_feature, T_op))
#define SN_MK_PROC_DECL(T_type, P_name) INTERNAL void SN_DECL P_name(MAT_TASK(T_type) const *task)
#define SN_MK_PROC_DECL_STD(T_type, T_arch, T_feature, T_op) \
INTERNAL NOINLINE SN_STD_TARGET_ATTR(T_arch, T_feature) void SN_DECL MK_PROC_NAME_STD( \
T_type, T_arch, T_feature, T_op)(MAT_TASK(T_type) const *task)
#pragma endregion
#pragma region sk_proc attributes
#define SK_PROC_NAME_STD(T_type, T_arch, T_feature, T_op) \
T_type##_single_kernel_proc_##T_op##_##T_arch##_##T_feature
#define GET_SK_PROC_STD(T_type, T_arch, T_feature, T_op) \
((UNI_SK_PROC)SK_PROC_NAME_STD(T_type, T_arch, T_feature, T_op))
#define SN_SK_PROC_DECL_STD(T_type, T_arch, T_feature, T_op) \
INTERNAL NOINLINE SN_STD_TARGET_ATTR(T_arch, T_feature) void SN_DECL SK_PROC_NAME_STD( \
T_type, T_arch, T_feature, \
T_op)(MAT_KERNEL(T_type) const *left, MAT_KERNEL(T_type) const *right, \
MAT_KERNEL(T_type) *restrict ans, uint64_t scalar)
#pragma endregion
#define _SN_STD__PROBE_0 0
#define _SN_STD__PROBE_1 1
#define SN_STD__PROBE_PRINT(x) _SN_STD__PROBE_##x
#define SN_STD__PROBE(x) SN_STD__PROBE_PRINT(x)
#pragma region param styles
#define _SN_MATPROC_LEFT_RIGHT_ANS 1
#define _SN_MATPROC_RIGHT_ANS 1
#define _SN_MATPROC_LEFT_RIGHT_ANS_SCALAR 1
#define _SN_MATPROC_RIGHT_ANS_SCALAR 1
#define _SN_MATPROC_ANS_SCALAR 1
#define _SN_MK_EXPAND_BUFFER_SIZE(i)\
int offset_##i = task->offset_##i;\
int count_##i = task->count_##i;\
#define _SN_MK_EXPAND_RIGHT_ANS \
SN_MAT_KERNEL_LOCAL const *right = (SN_MAT_KERNEL_LOCAL const *)task->buffer_1.ptr; \
SN_MAT_KERNEL_LOCAL *restrict ans = (SN_MAT_KERNEL_LOCAL *)task->buffer_2.ptr;\
_SN_MK_EXPAND_BUFFER_SIZE(1); \
_SN_MK_EXPAND_BUFFER_SIZE(2);
#define _SN_MK_EXPAND_LEFT_RIGHT_ANS \
SN_MAT_KERNEL_LOCAL const *left = (SN_MAT_KERNEL_LOCAL const *)task->buffer_0.ptr; \
SN_MAT_KERNEL_LOCAL const *right = (SN_MAT_KERNEL_LOCAL const *)task->buffer_1.ptr; \
SN_MAT_KERNEL_LOCAL *restrict ans = (SN_MAT_KERNEL_LOCAL *)task->buffer_2.ptr;\
_SN_MK_EXPAND_BUFFER_SIZE(0); \
_SN_MK_EXPAND_BUFFER_SIZE(1);\
_SN_MK_EXPAND_BUFFER_SIZE(2);
#define _SN_MK_EXPAND_RIGHT_ANS_SCALAR \
SN_MAT_KERNEL_LOCAL const *right = (SN_MAT_KERNEL_LOCAL const *)task->buffer_1.ptr; \
SN_MAT_KERNEL_LOCAL *restrict ans = (SN_MAT_KERNEL_LOCAL *)task->buffer_2.ptr; \
M128 scalar = task->scalar_pack;\
_SN_MK_EXPAND_BUFFER_SIZE(1);\
_SN_MK_EXPAND_BUFFER_SIZE(2);
#define _SN_MK_EXPAND_LEFT_RIGHT_ANS_SCALAR \
SN_MAT_KERNEL_LOCAL const *left = (SN_MAT_KERNEL_LOCAL const *)task->buffer_0.ptr; \
SN_MAT_KERNEL_LOCAL const *right = (SN_MAT_KERNEL_LOCAL const *)task->buffer_1.ptr; \
SN_MAT_KERNEL_LOCAL *restrict ans = (SN_MAT_KERNEL_LOCAL *)task->buffer_2.ptr; \
M128 scalar = task->scalar_pack;\
_SN_MK_EXPAND_BUFFER_SIZE(0);\
_SN_MK_EXPAND_BUFFER_SIZE(1);\
_SN_MK_EXPAND_BUFFER_SIZE(2);
#define _SN_MK_EXPAND_ANS_SCALAR \
SN_MAT_KERNEL_LOCAL *restrict ans = (SN_MAT_KERNEL_LOCAL *)task->buffer_2.ptr; \
M128 scalar = task->scalar_pack;\
_SN_MK_EXPAND_BUFFER_SIZE(2);
#define _SN_PARAM_STYLE(T) _SN_MATPROC_##T
#define _SN_MK_EXPAND_PARAM(style) _SN_MK_EXPAND_##style
#pragma endregion
#define __mk_param_std(param, T) \
typedef MAT_KERNEL(T) SN_MAT_KERNEL_LOCAL; \
matrix_task const *_ = (_SN_PARAM_STYLE(param), task); \
_SN_MK_EXPAND_PARAM(param)
#define __sk_param_std(T) \
typedef MAT_KERNEL(T) SN_MAT_KERNEL_LOCAL; \
(void)scalar
#endif // !SN_INTRINSIC_PARAM_EXPAND
+94
View File
@@ -0,0 +1,94 @@
#ifndef SN_TARGET
#define SN_TARGET
#include "sn_internal.h"
#include <stdint.h>
#define SN_OS_WINDOWS 0
#define SN_OS_LINUX 1
#define SN_OS_MACOS 2
#define SN_OS_IOS 3
#define SN_OS_ANDROID 4
#define SN_OS_UNKNOWN 5
#define SN_ARCH_X86_64 0
#define SN_ARCH_ARM64 1
#define SN_ARCH_RISCV64 2
#define SN_ARCH_PPC64le 3
#if defined(_M_X64) || defined(_M_AMD64) || defined(__x86_64__) || defined(__amd64__)
#define SN_CURRENT_ARCH SN_ARCH_X86_64
#elif defined(_M_ARM64) || defined(__aarch64__)
#define SN_CURRENT_ARCH SN_ARCH_ARM64
#elif defined(_M_ARM) || defined(__arm__)
#define SN_CURRENT_ARCH SN_ARCH_ARM32
#elif defined(_M_IX86) || defined(__i386__)
#define SN_CURRENT_ARCH SN_ARCH_X86_32
#elif defined(__powerpc64__) && defined(__LITTLE_ENDIAN__)
#define SN_CURRENT_ARCH SN_ARCH_PPC64LE
#elif defined(__riscv) && (__riscv_xlen == 64)
#define SN_CURRENT_ARCH SN_ARCH_RISCV64
#else
#error "SN: Unsupported target architecture"
#endif
/*
* Target selection policy:
* - Default (no SN_EXTERN_TARGET): fixed Windows + x86_64.
* - External (SN_EXTERN_TARGET): build system must provide SN_OS and SN_ARCH.
*/
#ifndef SN_EXTERN_TARGET
#define SN_OS_TARGET SN_OS_WINDOWS
#define SN_ARCH_TARGET SN_ARCH_X86_64
#else
#ifndef SN_OS_TARGET
#error "SN_EXTERN_TARGET is defined but SN_OS is not provided."
#endif
#ifndef SN_ARCH_TARGET
#error "SN_EXTERN_TARGET is defined but SN_ARCH is not provided."
#endif
/* Validate externally supplied values are within known enum constants. */
#if (SN_OS_TARGET != SN_OS_WINDOWS) && (SN_OS_TARGET != SN_OS_LINUX) && \
(SN_OS_TARGET != SN_OS_MACOS) && (SN_OS_TARGET != SN_OS_IOS) && \
(SN_OS_TARGET != SN_OS_ANDROID) && (SN_OS_TARGET != SN_OS_UNKNOWN)
#error "SN_OS has an invalid value. Use SN_OS_<...> enum constants."
#endif
#if (SN_ARCH_TARGET != SN_ARCH_X86_64) && (SN_ARCH_TARGET != SN_ARCH_ARM64) && \
(SN_ARCH_TARGET != SN_ARCH_RISCV64) && (SN_ARCH_TARGET != SN_ARCH_PPC64le)
#error "SN_ARCH has an invalid value. Use SN_ARCH_<...> enum constants."
#endif
#endif
#define SN_IS_OS(os) ((SN_OS_TARGET) == (os))
#define SN_IS_ARCH(arch) ((SN_ARCH_TARGET) == (arch))
/*
* Per-function ISA selection is required for runtime-dispatched kernels that
* mix multiple x86 feature levels in one build target.
*/
#if defined(__clang__) && ((SN_ARCH_TARGET) == SN_ARCH_X86_64)
#define SN_CLANG_TARGET_ATTR(options) __attribute__((target(options)))
#else
#define SN_CLANG_TARGET_ATTR(options)
#endif
typedef union SIMDID {
uint64_t mask;
char make_byte[8];
} SIMDID;
#endif // SN_TARGET
+42
View File
@@ -0,0 +1,42 @@
#include "StgSharpNative.h"
#include "sn_intrinsic.matkernel.h"
#include <stdint.h>
#ifdef _WIN32
/* Define and initialize a global zero kernel aligned to 64 bytes for universal ISA usage */
__declspec(align(64)) MAT_KERNEL(u64) SN_ZERO_KERNEL;
static void init_zero_kernel(void)
{
for (int c = 0; c < 4; ++c) {
for (int r = 0; r < 4; ++r) {
SN_ZERO_KERNEL.m[r][c] = 0;
}
}
}
BOOL APIENTRY DllMain(HMODULE hModule, DWORD ul_reason_for_call, LPVOID lpReserved)
{
switch (ul_reason_for_call) {
case DLL_PROCESS_ATTACH:
init_zero_kernel();
return 1;
case DLL_THREAD_ATTACH:
return 1;
case DLL_THREAD_DETACH:
return 1;
case DLL_PROCESS_DETACH:
return 1;
break;
}
return TRUE;
}
#else
attribute((constructor)) void DllMain()
{
init_zero_kernel();
}
#endif // _WIN32
+121
View File
@@ -0,0 +1,121 @@
#include "StgSharpNative.h"
SN_API void SN_DECL load_glfw_functions(GLFWFunctionTable *table)
{
table->glfwCreateCursor = glfwCreateCursor;
table->glfwCreateStandardCursor = glfwCreateStandardCursor;
table->glfwCreateWindow = glfwCreateWindow;
table->glfwDefaultWindowHints = glfwDefaultWindowHints;
table->glfwDestroyCursor = glfwDestroyCursor;
table->glfwDestroyWindow = glfwDestroyWindow;
table->glfwExtensionSupported = glfwExtensionSupported;
table->glfwFocusWindow = glfwFocusWindow;
table->glfwGetClipboardString = glfwGetClipboardString;
table->glfwGetCurrentContext = glfwGetCurrentContext;
table->glfwGetCursorPos = glfwGetCursorPos;
table->glfwGetError = glfwGetError;
table->glfwGetFrameBufferSize = glfwGetFramebufferSize;
table->glfwGetGamepadName = glfwGetGamepadName;
table->glfwGetGamepadState = glfwGetGamepadState;
table->glfwGetGammaRamp = glfwGetGammaRamp;
table->glfwGetInputMode = glfwGetInputMode;
table->glfwGetJoystickAxes = glfwGetJoystickAxes;
table->glfwGetJoystickButtons = glfwGetJoystickButtons;
table->glfwGetJoystickGUID = glfwGetJoystickGUID;
table->glfwGetJoystickHats = glfwGetJoystickHats;
table->glfwGetJoystickName = glfwGetJoystickName;
table->glfwGetJoystickUserPointer = glfwGetJoystickUserPointer;
table->glfwGetKey = glfwGetKey;
table->glfwGetKeyName = glfwGetKeyName;
table->glfwGetKeyScancode = glfwGetKeyScancode;
table->glfwGetMonitorContentScale = glfwGetMonitorContentScale;
table->glfwGetMonitorName = glfwGetMonitorName;
table->glfwGetMonitorPhysicalSize = glfwGetMonitorPhysicalSize;
table->glfwGetMonitorPos = glfwGetMonitorPos;
table->glfwGetMonitors = glfwGetMonitors;
table->glfwGetMonitorUserPointer = glfwGetMonitorUserPointer;
table->glfwGetMonitorWorkarea = glfwGetMonitorWorkarea;
table->glfwGetMouseButton = glfwGetMouseButton;
table->glfwGetPrimaryMonitor = glfwGetPrimaryMonitor;
table->glfwGetProcAddress = glfwGetProcAddress;
table->glfwGetRequiredInstanceExtensions = glfwGetRequiredInstanceExtensions;
table->glfwGetTime = glfwGetTime;
table->glfwGetTimerFrequency = glfwGetTimerFrequency;
table->glfwGetTimerValue = glfwGetTimerValue;
table->glfwGetVersion = glfwGetVersion;
table->glfwGetVersionString = glfwGetVersionString;
table->glfwGetVideoMode = glfwGetVideoMode;
table->glfwGetVideoModes = glfwGetVideoModes;
table->glfwGetWindowAttrib = glfwGetWindowAttrib;
table->glfwGetWindowContentScale = glfwGetWindowContentScale;
table->glfwGetWindowFrameSize = glfwGetWindowFrameSize;
table->glfwGetWindowMonitor = glfwGetWindowMonitor;
table->glfwGetWindowOpacity = glfwGetWindowOpacity;
table->glfwGetWindowPos = glfwGetWindowPos;
table->glfwGetWindowSize = glfwGetWindowSize;
table->glfwGetWindowUserPointer = glfwGetWindowUserPointer;
table->glfwHideWindow = glfwHideWindow;
table->glfwIconifyWindow = glfwIconifyWindow;
table->glfwInit = glfwInit;
table->glfwInitHint = glfwInitHint;
table->glfwJoystickIsGamePad = glfwJoystickIsGamepad;
table->glfwJoystickPresent = glfwJoystickPresent;
table->glfwMakeContextCurrent = glfwMakeContextCurrent;
table->glfwMaximizeWindow = glfwMaximizeWindow;
table->glfwPollEvents = glfwPollEvents;
table->glfwPostEmptyEvent = glfwPostEmptyEvent;
table->glfwRawMouseMotionSupported = glfwRawMouseMotionSupported;
table->glfwRequestWindowAttention = glfwRequestWindowAttention;
table->glfwRestoreWindow = glfwRestoreWindow;
table->glfwSetCharCallback = glfwSetCharCallback;
table->glfwSetCharModsCallback = glfwSetCharModsCallback;
table->glfwSetClipboardString = glfwSetClipboardString;
table->glfwSetCursor = glfwSetCursor;
table->glfwSetCursorEnterCallback = glfwSetCursorEnterCallback;
table->glfwSetCursorPos = glfwSetCursorPos;
table->glfwSetCursorPosCallback = glfwSetCursorPosCallback;
table->glfwSetDropCallback = glfwSetDropCallback;
table->glfwSetErrorCallback = glfwSetErrorCallback;
table->glfwSetFramebufferSizeCallback = glfwSetFramebufferSizeCallback;
table->glfwSetGamma = glfwSetGamma;
table->glfwSetGammaRamp = glfwSetGammaRamp;
table->glfwSetInputMode = glfwSetInputMode;
table->glfwSetJoystickCallback = glfwSetJoystickCallback;
table->glfwSetJoystickUserPointer = glfwSetJoystickUserPointer;
table->glfwSetKeyCallback = glfwSetKeyCallback;
table->glfwSetMonitorCallback = glfwSetMonitorCallback;
table->glfwSetMonitorUserPointer = glfwSetMonitorUserPointer;
table->glfwSetMouseButtonCallback = glfwSetMouseButtonCallback;
table->glfwSetScrollCallback = glfwSetScrollCallback;
table->glfwSetTime = glfwSetTime;
table->glfwSetWindowAspectRatio = glfwSetWindowAspectRatio;
table->glfwSetWindowAttrib = glfwSetWindowAttrib;
table->glfwSetWindowCloseCallback = glfwSetWindowCloseCallback;
table->glfwSetWindowContentScaleCallback = glfwSetWindowContentScaleCallback;
table->glfwSetWindowFocusCallback = glfwSetWindowFocusCallback;
table->glfwSetWindowIcon = glfwSetWindowIcon;
table->glfwSetWindowIconifyCallback = glfwSetWindowIconifyCallback;
table->glfwSetWindowMaximizeCallback = glfwSetWindowMaximizeCallback;
table->glfwSetWindowMonitor = glfwSetWindowMonitor;
table->glfwSetWindowOpacity = glfwSetWindowOpacity;
table->glfwSetWindowPos = glfwSetWindowPos;
table->glfwSetWindowPosCallback = glfwSetWindowPosCallback;
table->glfwSetWindowRefreshCallback = glfwSetWindowRefreshCallback;
table->glfwSetWindowShouldClose = glfwSetWindowShouldClose;
table->glfwSetWindowSize = glfwSetWindowSize;
table->glfwSetWindowSizeCallback = glfwSetWindowSizeCallback;
table->glfwSetWindowSizeLimits = glfwSetWindowSizeLimits;
table->glfwSetWindowTitle = glfwSetWindowTitle;
table->glfwSetWindowUserPointer = glfwSetWindowUserPointer;
table->glfwShowWindow = glfwShowWindow;
table->glfwSwapBuffers = glfwSwapBuffers;
table->glfwSwapInterval = glfwSwapInterval;
table->glfwTerminate = glfwTerminate;
table->glfwUpdateGamePadMappings = glfwUpdateGamepadMappings;
table->glfwVulkanSupported = glfwVulkanSupported;
table->glfwWaitEvents = glfwWaitEvents;
table->glfwWaitEventsTimeout = glfwWaitEventsTimeout;
table->glfwWindowHint = glfwWindowHint;
table->glfwWindowHintString = glfwWindowHintString;
table->glfwWindowShouldClose = glfwWindowShouldClose;
}
@@ -0,0 +1,118 @@
#include "sn_target.h"
#if SN_IS_ARCH(SN_ARCH_X86_64)
#include "StgSharpNative.h"
#include "sn_intrinsic.h"
#include "sn_intrinsic.std.h"
#include "sn_intrinsic.context.x86.h"
PRIVATE void load_intrinsic_sse(sn_intrinsic *context, SIMDID id);
PRIVATE void load_intrinsic_avx(sn_intrinsic *context, SIMDID id);
PRIVATE void load_intrinsic_512(sn_intrinsic *context, SIMDID id);
SN_API void SN_DECL load_intrinsic_function(void *intrinsic_context, uint64_t id)
{
sn_null_assert(intrinsic_context);
SIMDID simdid;
if (id) {
simdid.mask = id;
} else {
simdid = sn_get_simd_level_global();
}
sn_intrinsic *context = (sn_intrinsic *)intrinsic_context;
switch (simdid.make_byte[1]) {
case SIMDID_MAIN_LVL_SSE:
load_intrinsic_sse(context, simdid);
break;
case SIMDID_MAIN_LVL_AVX2:
load_intrinsic_avx(context, simdid);
break;
case SIMDID_MAIN_LVL_AVX512:
case SIMDID_MAIN_LVL_AVX10:
load_intrinsic_512(context, simdid);
break;
}
}
void load_intrinsic_sse(sn_intrinsic *context, SIMDID id)
{
context->city_hash_simplify = city_hash_simplify_sse;
context->factorial_simd = factorial_simd_sse;
context->mat_mk[F32].proc_array[Add] = GET_MK_PROC_STD(float, sse, , add);
context->mat_mk[F32].proc_array[Fill] = GET_MK_PROC_STD(float, sse, , fill);
context->mat_mk[F32].proc_array[ScalarMul] = GET_MK_PROC_STD(float, sse, , scalar_mul);
context->mat_mk[F32].proc_array[Sub] = GET_MK_PROC_STD(float, sse, , sub);
context->mat_mk[F32].proc_array[Transpose] = GET_MK_PROC_STD(float, sse, , transpose);
context->mat_mk[F32].proc_array[Fma] = GET_MK_PROC_STD(float, sse, , fma);
context->mat_mk[F32].proc_array[Pivot] = NULL;
context->mat_mk[F64].proc_array[Add] = GET_MK_PROC_STD(double, sse, , add);
context->mat_mk[F64].proc_array[Fill] = GET_MK_PROC_STD(double, sse, , fill);
context->mat_mk[F64].proc_array[ScalarMul] = GET_MK_PROC_STD(double, sse, , scalar_mul);
context->mat_mk[F64].proc_array[Sub] = GET_MK_PROC_STD(double, sse, , sub);
context->mat_mk[F64].proc_array[Transpose] = NULL;
context->mat_mk[F64].proc_array[Fma] = GET_MK_PROC_STD(double, sse, , fma);
context->mat_mk[F64].proc_array[Pivot] = NULL;
context->vec_f32_normalize_3 = f32_normalize;
}
void load_intrinsic_avx(sn_intrinsic *context, SIMDID id)
{
char avx_seg = id.make_byte[4];
char is_fma = (avx_seg & SIMDID_AVX_FMA) != 0;
context->city_hash_simplify = city_hash_simplify_sse;
context->factorial_simd = factorial_simd_sse;
context->mat_mk[F32].proc_array[Add] = GET_MK_PROC_STD(float, avx, , add);
context->mat_mk[F32].proc_array[Fill] = GET_MK_PROC_STD(float, avx, , fill);
context->mat_mk[F32].proc_array[ScalarMul] = GET_MK_PROC_STD(float, avx, , scalar_mul);
context->mat_mk[F32].proc_array[Sub] = GET_MK_PROC_STD(float, avx, , sub);
context->mat_mk[F32].proc_array[Transpose] = GET_MK_PROC_STD(float, avx, , transpose);
context->mat_mk[F32].proc_array[Fma] = is_fma ? GET_MK_PROC_STD(float, avx, _fma, fma) :
GET_MK_PROC_STD(float, avx, , fma);
context->mat_mk[F32].proc_array[Pivot] = NULL;
context->mat_mk[F64].proc_array[Add] = GET_MK_PROC_STD(double, avx, , add);
context->mat_mk[F64].proc_array[Fill] = GET_MK_PROC_STD(double, avx, , fill);
context->mat_mk[F64].proc_array[ScalarMul] = GET_MK_PROC_STD(double, avx, , scalar_mul);
context->mat_mk[F64].proc_array[Sub] = GET_MK_PROC_STD(double, avx, , sub);
context->mat_mk[F64].proc_array[Transpose] = NULL;
context->mat_mk[F64].proc_array[Fma] = is_fma ? GET_MK_PROC_STD(double, avx, _fma, fma) :
GET_MK_PROC_STD(double, avx, , fma);
context->mat_mk[F64].proc_array[Pivot] = NULL;
context->vec_f32_normalize_3 = f32_normalize;
}
void load_intrinsic_512(sn_intrinsic *context, SIMDID id)
{
context->city_hash_simplify = city_hash_simplify_sse;
context->factorial_simd = factorial_simd_sse;
context->mat_mk[F32].proc_array[Add] = GET_MK_PROC_STD(float, 512, , add);
context->mat_mk[F32].proc_array[Fill] = GET_MK_PROC_STD(float, 512, , fill);
context->mat_mk[F32].proc_array[ScalarMul] = GET_MK_PROC_STD(float, 512, , scalar_mul);
context->mat_mk[F32].proc_array[Sub] = GET_MK_PROC_STD(float, 512, , sub);
context->mat_mk[F32].proc_array[Transpose] = GET_MK_PROC_STD(float, 512, , transpose);
context->mat_mk[F32].proc_array[Fma] = GET_MK_PROC_STD(float, 512, , fma);
context->mat_mk[F32].proc_array[Pivot] = NULL;
context->mat_mk[F64].proc_array[Add] = GET_MK_PROC_STD(double, 512, , add);
context->mat_mk[F64].proc_array[Fill] = GET_MK_PROC_STD(double, 512, , fill);
context->mat_mk[F64].proc_array[ScalarMul] = GET_MK_PROC_STD(double, 512, , scalar_mul);
context->mat_mk[F64].proc_array[Sub] = GET_MK_PROC_STD(double, 512, , sub);
context->mat_mk[F64].proc_array[Transpose] = NULL;
context->mat_mk[F64].proc_array[Fma] = GET_MK_PROC_STD(double, 512, , fma);
context->mat_mk[F64].proc_array[Pivot] = NULL;
context->vec_f32_normalize_3 = f32_normalize;
}
#endif
+369
View File
@@ -0,0 +1,369 @@
#include "sn_target.h"
#if SN_IS_ARCH(SN_ARCH_X86_64)
#include <immintrin.h>
#include "sn_intrinsic.h"
#include "sn_intrinsic.std.h"
#include "sn_intrinsic.context.x86.h"
SN_MK_PROC_DECL_STD(float, sse, , fill)
{
__mk_param_std(ANS_SCALAR, float);
register __m128 vec = _mm_set_ps1(scalar.f32[0]);
for (size_t i = 0; i < count_2; i++) {
ans[i].x[0] = vec;
ans[i].x[1] = vec;
ans[i].x[2] = vec;
ans[i].x[3] = vec;
}
}
SN_MK_PROC_DECL_STD(float, avx, , fill)
{
__mk_param_std(ANS_SCALAR, float);
register __m256 vec = _mm256_broadcast_ss(&scalar.f32[0]);
for (size_t i = 0; i < count_2; i++) {
ans[i].y[0] = vec;
ans[i].y[1] = vec;
}
}
SN_MK_PROC_DECL_STD(float, 512, , fill)
{
__mk_param_std(ANS_SCALAR, float);
register __m512 vec = _mm512_broadcastss_ps(scalar.m128);
for (size_t i = 0; i < count_2; i++) {
ans[i].z[0] = vec;
}
}
SN_MK_PROC_DECL_STD(double, sse, , fill)
{
__mk_param_std(ANS_SCALAR, double);
register __m128d vec_d = _mm_set1_pd(scalar.f64[0]);
register __m128 vec = _mm_castpd_ps(vec_d);
for (size_t i = 0; i < count_2; i++) {
ans[i].x[0] = vec;
ans[i].x[1] = vec;
ans[i].x[2] = vec;
ans[i].x[3] = vec;
ans[i].x[4] = vec;
ans[i].x[5] = vec;
ans[i].x[6] = vec;
ans[i].x[7] = vec;
}
}
SN_MK_PROC_DECL_STD(double, avx, , fill)
{
__mk_param_std(ANS_SCALAR, double);
register __m256d vec = _mm256_broadcast_sd(&scalar.f64[0]);
for (size_t i = 0; i < count_2; i++) {
ans[i].y[0] = _mm256_castpd_ps(vec);
ans[i].y[1] = _mm256_castpd_ps( vec);
ans[i].y[2] = _mm256_castpd_ps( vec);
ans[i].y[3] = _mm256_castpd_ps( vec);
}
}
SN_MK_PROC_DECL_STD(double, 512, , fill)
{
__mk_param_std(ANS_SCALAR, double);
register __m512d vec = _mm512_set1_pd(scalar.f64[0]);
for (size_t i = 0; i < count_2; i++) {
ans[i].z[0] = _mm512_castpd_ps(vec);
ans[i].z[1] = _mm512_castpd_ps(vec);
}
}
SN_MK_PROC_DECL_STD(float, sse, , add)
{
__mk_param_std(LEFT_RIGHT_ANS, float);
for (size_t i = 0; i < count_0; i++) {
register __m128 c0 = _mm_add_ps(left[i].x[0], right[i].x[0]);
register __m128 c1 = _mm_add_ps(left[i].x[1], right[i].x[1]);
register __m128 c2 = _mm_add_ps(left[i].x[2], right[i].x[2]);
register __m128 c3 = _mm_add_ps(left[i].x[3], right[i].x[3]);
ans[i].x[0] = c0;
ans[i].x[1] = c1;
ans[i].x[2] = c2;
ans[i].x[3] = c3;
}
}
SN_MK_PROC_DECL_STD(float, avx, , add)
{
__mk_param_std(LEFT_RIGHT_ANS, float);
for (size_t i = 0; i < count_0; i++) {
size_t s = sizeof(MAT_KERNEL(float));
register __m256 c0 = _mm256_add_ps(left[i].y[0], right[i].y[0]);
register __m256 c1 = _mm256_add_ps(left[i].y[1], right[i].y[1]);
ans[i].y[0] = c0;
ans[i].y[1] = c1;
}
}
SN_MK_PROC_DECL_STD(float, 512, , add)
{
__mk_param_std(LEFT_RIGHT_ANS, float);
for (size_t i = 0; i < count_0; i++) {
register __m512 c0 = _mm512_add_ps(left[i].z[0], right[i].z[0]);
ans[i].z[0] = c0;
}
}
SN_MK_PROC_DECL_STD(double, sse, , add)
{
__mk_param_std( LEFT_RIGHT_ANS, double);
for (size_t i = 0; i < count_0; i++) {
register __m128d c0 = _mm_add_pd(_mm_castps_pd(left[i].x[0]),
_mm_castps_pd(right[i].x[0]));
register __m128d c1 = _mm_add_pd(_mm_castps_pd(left[i].x[1]),
_mm_castps_pd(right[i].x[1]));
register __m128d c2 = _mm_add_pd(_mm_castps_pd(left[i].x[2]),
_mm_castps_pd(right[i].x[2]));
register __m128d c3 = _mm_add_pd(_mm_castps_pd(left[i].x[3]),
_mm_castps_pd(right[i].x[3]));
register __m128d c4 = _mm_add_pd(_mm_castps_pd(left[i].x[4]),
_mm_castps_pd(right[i].x[4]));
register __m128d c5 = _mm_add_pd(_mm_castps_pd(left[i].x[5]),
_mm_castps_pd(right[i].x[5]));
register __m128d c6 = _mm_add_pd(_mm_castps_pd(left[i].x[6]),
_mm_castps_pd(right[i].x[6]));
register __m128d c7 = _mm_add_pd(_mm_castps_pd(left[i].x[7]),
_mm_castps_pd(right[i].x[7]));
ans[i].x[0] = _mm_castpd_ps(c0);
ans[i].x[1] = _mm_castpd_ps(c1);
ans[i].x[2] = _mm_castpd_ps(c2);
ans[i].x[3] = _mm_castpd_ps(c3);
ans[i].x[4] = _mm_castpd_ps(c4);
ans[i].x[5] = _mm_castpd_ps(c5);
ans[i].x[6] = _mm_castpd_ps(c6);
ans[i].x[7] = _mm_castpd_ps(c7);
}
}
SN_MK_PROC_DECL_STD(double, avx, , add)
{
__mk_param_std( LEFT_RIGHT_ANS, double);
for (size_t i = 0; i < count_0; i++) {
size_t s = sizeof(MAT_KERNEL(float));
register __m256d c0 = _mm256_add_pd(left[i].y[0], right[i].y[0]);
register __m256d c1 = _mm256_add_pd(left[i].y[1], right[i].y[1]);
register __m256d c2 = _mm256_add_pd(left[i].y[0], right[i].y[0]);
register __m256d c3 = _mm256_add_pd(left[i].y[1], right[i].y[1]);
ans[i].y[0] = c0;
ans[i].y[1] = c1;
ans[i].y[2] = c2;
ans[i].y[3] = c3;
}
}
SN_MK_PROC_DECL_STD(double, 512, , add)
{
__mk_param_std( LEFT_RIGHT_ANS, double);
for (size_t i = 0; i < count_0; i++) {
register __m512d c0 = _mm512_add_pd(_mm512_castps_pd( left[i].z[0]), _mm512_castps_pd( right[i].z[0]));
register __m512d c1 = _mm512_add_pd(_mm512_castps_pd( left[i].z[1]), _mm512_castps_pd( right[i].z[1]));
ans[i].z[0] = c0;
ans[i].z[1] = c1;
}
}
SN_MK_PROC_DECL_STD(float, sse, , sub)
{
__mk_param_std( LEFT_RIGHT_ANS, float);
for (size_t i = 0; i < count_0; i++) {
register __m128 c0 = _mm_sub_ps(left[i].x[0], right[i].x[0]);
register __m128 c1 = _mm_sub_ps(left[i].x[1], right[i].x[1]);
register __m128 c2 = _mm_sub_ps(left[i].x[2], right[i].x[2]);
register __m128 c3 = _mm_sub_ps(left[i].x[3], right[i].x[3]);
ans[i].x[0] = c0;
ans[i].x[1] = c1;
ans[i].x[2] = c2;
ans[i].x[3] = c3;
}
}
SN_MK_PROC_DECL_STD(float, avx, , sub)
{
__mk_param_std( LEFT_RIGHT_ANS, float);
for (size_t i = 0; i < count_0; i++) {
register __m256 c0 = _mm256_sub_ps(left[i].y[0], right[i].y[0]);
register __m256 c1 = _mm256_sub_ps(left[i].y[1], right[i].y[1]);
ans[i].y[0] = c0;
ans[i].y[1] = c1;
}
}
SN_MK_PROC_DECL_STD(float, 512, , sub)
{
__mk_param_std( LEFT_RIGHT_ANS, float);
for (size_t i = 0; i < count_0; i++) {
register __m512 c0 = _mm512_sub_ps(left[i].z[0], right[i].z[0]);
ans[i].z[0] = c0;
}
}
SN_MK_PROC_DECL_STD(double, sse, , sub)
{
__mk_param_std( LEFT_RIGHT_ANS, double);
for (size_t i = 0; i < count_0; i++) {
register __m128d c0 = _mm_sub_pd(left[i].x[0], right[i].x[0]);
register __m128d c1 = _mm_sub_pd(left[i].x[1], right[i].x[1]);
register __m128d c2 = _mm_sub_pd(left[i].x[2], right[i].x[2]);
register __m128d c3 = _mm_sub_pd(left[i].x[3], right[i].x[3]);
register __m128d c4 = _mm_sub_pd(left[i].x[4], right[i].x[4]);
register __m128d c5 = _mm_sub_pd(left[i].x[5], right[i].x[5]);
register __m128d c6 = _mm_sub_pd(left[i].x[6], right[i].x[6]);
register __m128d c7 = _mm_sub_pd(left[i].x[7], right[i].x[7]);
ans[i].x[0] = c0;
ans[i].x[1] = c1;
ans[i].x[2] = c2;
ans[i].x[3] = c3;
ans[i].x[4] = c4;
ans[i].x[5] = c5;
ans[i].x[6] = c6;
ans[i].x[7] = c7;
}
}
SN_MK_PROC_DECL_STD(double, avx, , sub)
{
__mk_param_std( LEFT_RIGHT_ANS, double);
for (size_t i = 0; i < count_0; i++) {
register __m256 c0 = _mm256_sub_ps(left[i].y[0], right[i].y[0]);
register __m256 c1 = _mm256_sub_ps(left[i].y[1], right[i].y[1]);
ans[i].y[0] = c0;
ans[i].y[1] = c1;
}
}
SN_MK_PROC_DECL_STD(double, 512, , sub)
{
__mk_param_std( LEFT_RIGHT_ANS, double);
for (size_t i = 0; i < count_0; i++) {
register __m512 c0 = _mm512_sub_ps(left[i].z[0], right[i].z[0]);
ans[i].z[0] = c0;
}
}
SN_MK_PROC_DECL_STD(float, sse, , scalar_mul)
{
__mk_param_std( RIGHT_ANS_SCALAR, float);
register __m128 vec = _mm_set_ps1(scalar.f32[0]);
for (size_t i = 0; i < count_1; i++) {
register __m128 c0 = _mm_mul_ps(right[i].x[0], vec);
register __m128 c1 = _mm_mul_ps(right[i].x[1], vec);
register __m128 c2 = _mm_mul_ps(right[i].x[2], vec);
register __m128 c3 = _mm_mul_ps(right[i].x[3], vec);
ans[i].x[0] = c0;
ans[i].x[1] = c1;
ans[i].x[2] = c2;
ans[i].x[3] = c3;
}
}
SN_MK_PROC_DECL_STD(float, avx, , scalar_mul)
{
__mk_param_std( RIGHT_ANS_SCALAR, float);
register __m256 vec = _mm256_broadcast_ss(&scalar.f32[0]);
for (size_t i = 0; i < count_1; i++) {
register __m256 c0 = _mm256_mul_ps(right[i].y[0], vec);
register __m256 c1 = _mm256_mul_ps(right[i].y[1], vec);
ans[i].y[0] = c0;
ans[i].y[1] = c1;
}
}
SN_MK_PROC_DECL_STD(float, 512, , scalar_mul)
{
__mk_param_std( RIGHT_ANS_SCALAR, float);
register __m512 vec = _mm512_broadcastss_ps(scalar.m128);
for (size_t i = 0; i < count_1; i++) {
register __m512 c0 = _mm512_mul_ps(right[i].z[0], vec);
ans[i].z[0] = c0;
}
}
SN_MK_PROC_DECL_STD(double, sse, , scalar_mul)
{
__mk_param_std( RIGHT_ANS_SCALAR, double);
register __m128d vec = _mm_set1_pd(scalar.f64[0]);
for (size_t i = 0; i < count_1; i++) {
register __m128d c0 = _mm_mul_pd(right[i].x[0], vec);
register __m128d c1 = _mm_mul_pd(right[i].x[1], vec);
register __m128d c2 = _mm_mul_pd(right[i].x[2], vec);
register __m128d c3 = _mm_mul_pd(right[i].x[3], vec);
register __m128d c4 = _mm_mul_pd(right[i].x[4], vec);
register __m128d c5 = _mm_mul_pd(right[i].x[5], vec);
register __m128d c6 = _mm_mul_pd(right[i].x[6], vec);
register __m128d c7 = _mm_mul_pd(right[i].x[7], vec);
ans[i].x[0] = c0;
ans[i].x[1] = c1;
ans[i].x[2] = c2;
ans[i].x[3] = c3;
ans[i].x[4] = c4;
ans[i].x[5] = c5;
ans[i].x[6] = c6;
ans[i].x[7] = c7;
}
}
SN_MK_PROC_DECL_STD(double, avx, , scalar_mul)
{
__mk_param_std( RIGHT_ANS_SCALAR, double);
register __m256d vec = _mm256_broadcast_sd(&scalar.f64[0]);
for (size_t i = 0; i < count_1; i++) {
register __m256d c0 = _mm256_mul_pd(right[i].y[0], vec);
register __m256d c1 = _mm256_mul_pd(right[i].y[1], vec);
register __m256d c2 = _mm256_mul_pd(right[i].y[2], vec);
register __m256d c3 = _mm256_mul_pd(right[i].y[3], vec);
ans[i].y[0] = c0;
ans[i].y[1] = c1;
ans[i].y[2] = c2;
ans[i].y[3] = c3;
}
}
SN_MK_PROC_DECL_STD(double, 512, , scalar_mul)
{
__mk_param_std( RIGHT_ANS_SCALAR, double);
register __m512d vec = _mm512_set1_pd(scalar.f64[0]);
for (size_t i = 0; i < count_1; i++) {
register __m512d c0 = _mm512_mul_pd(right[i].z[0], vec);
register __m512d c1 = _mm512_mul_pd(right[i].z[1], vec);
ans[i].z[0] = c0;
ans[i].z[1] = c1;
}
}
#endif
+184
View File
@@ -0,0 +1,184 @@
#include "sn_target.h"
#if SN_IS_ARCH(SN_ARCH_X86_64)
#include "sn_intrinsic.h"
#include "sn_intrinsic.std.h"
#include "sn_intrinsic.context.x86.h"
#include <math.h>
static const float quality_threshold = 1e-6f;
SN_MK_PROC_DECL_STD(float, sse, , try_plu)
{
__mk_param_std(LEFT_RIGHT_ANS_SCALAR, float);
/* Aliases */
MAT_KERNEL(float) *upper = left; /* U */
MAT_KERNEL(float) *lower = right; /* L (unit diag) */
MAT_KERNEL(float) *source = ans; /* A */
unsigned char *pivot = (unsigned char *)&scalar.i64[0]; /* length = 4 */
int *success = (int *)&scalar.i64[1];
/* Init permutation and outputs */
pivot[0] = 0;
pivot[1] = 1;
pivot[2] = 2;
pivot[3] = 3;
*success = 1;
/* Working copy of A in column-major a[col][row], load via SIMD view */
MAT_KERNEL(float) a;
const __m128 z = _mm_setzero_ps();
for (int c = 0; c < 4; ++c) {
__m128 col = _mm_load_ps((float *)&source->x[c]);
_mm_storeu_ps((float *)&a.x[c], col);
_mm_store_ps((float *)&upper->x[c], z);
_mm_store_ps((float *)&lower->x[c], z);
}
const float eps = 1e-12f;
for (int k = 0; k < 4; ++k) {
/* Pivot search on column k, rows k..3 */
int p = k;
float maxabs = fabsf(a.m[k][p]);
for (int r = k + 1; r < 4; ++r) {
float v = fabsf(a.m[k][r]);
if (v > maxabs) {
maxabs = v;
p = r;
}
}
if (maxabs < eps) {
*success = 0;
/* leave outputs zeroed */
return;
}
/* Swap rows if needed */
if (p != k) {
for (int c = 0; c < 4; ++c) {
float tmp = a.m[c][k];
a.m[c][k] = a.m[c][p];
a.m[c][p] = tmp;
}
unsigned char tmpi = pivot[k];
pivot[k] = pivot[p];
pivot[p] = tmpi;
}
float pivot_val = a.m[k][k];
/* Eliminate below pivot: store multipliers in L column k */
for (int r = k + 1; r < 4; ++r) {
float factor = a.m[k][r] / pivot_val;
lower->m[k][r] = factor;
for (int c = k; c < 4; ++c) {
a.m[c][r] -= factor * a.m[c][k];
}
}
/* Set U row k (columns k..3) and L diagonal */
lower->m[k][k] = 1.0f;
for (int c = k; c < 4; ++c) {
upper->m[c][k] = a.m[c][k];
}
}
/* Ensure remaining diagonal ones for L */
for (int d = 0; d < 4; ++d) {
if (lower->m[d][d] == 0.0f)
lower->m[d][d] = 1.0f;
}
}
#define DOT_F32_SSE(a, b, ans) \
do { \
__m128 _p = _mm_mul_ps((a), (b)); /* p0 p1 p2 p3 */ \
__m128 _t = _mm_shuffle_ps(_p, _p, _MM_SHUFFLE(2, 3, 0, 1)); \
_p = _mm_add_ps(_p, _t); \
_t = _mm_movehl_ps(_t, _p); \
ans = _mm_cvtss_f32(_mm_add_ss(_p, _t)); \
} while (0)
#define FIND_MAX_F32_SSE(a, b) \
do { \
__m128 _t = _mm_movehl_ps(a, a); \
a = _mm_max_ps(a, _t); \
_t = _mm_shuffle_ps(a, a, _MM_SHUFFLE(2, 3, 0, 1)); \
b = _mm_max_ps(a, _t); \
} while (0)
/*
* layout of scalarpack:
*
*/
SN_MK_PROC_DECL_STD(float, sse, , quality)
{
__mk_param_std(RIGHT_ANS, float);
register __m128 tiny = _mm_set1_ps(1e-6f);
register __m128 col0 = _mm_load_ps((float *)&right->x[0]);
register __m128 col1 = _mm_load_ps((float *)&right->x[1]);
register __m128 col2 = _mm_load_ps((float *)&right->x[2]);
register __m128 col3 = _mm_load_ps((float *)&right->x[3]);
SN_ALIGNAS(16) float self_dot_src[4] = { 0.0f, 0.0f, 0.0f, 0.0f };
// try in reigister mix later
DOT_F32_SSE(col0, col0, self_dot_src[0]);
DOT_F32_SSE(col1, col1, self_dot_src[1]);
DOT_F32_SSE(col2, col2, self_dot_src[2]);
DOT_F32_SSE(col3, col3, self_dot_src[3]);
register __m128 self_dot = _mm_load_ps(self_dot_src);
register __m128 max;
FIND_MAX_F32_SSE(self_dot, max);
//float norm = _mm_cvtss_f32(max);
// min on cross
register __m128 col_shuff_1 = _mm_shuffle_ps(col1, col1, _MM_SHUFFLE(0, 3, 2, 1));
register __m128 col_shuff_2 = _mm_shuffle_ps(col2, col2, _MM_SHUFFLE(1, 0, 3, 2));
register __m128 col_shuff_3 = _mm_shuffle_ps(col3, col3, _MM_SHUFFLE(2, 1, 0, 3));
register __m128 min = _mm_min_ps(col0, col_shuff_1);
register __m128 min0 = _mm_min_ps(col_shuff_2, col_shuff_3);
min = _mm_min_ps(min, min0);
float quality = _mm_cvtss_f32(_mm_div_ps(min, _mm_add_ps(max, tiny)));
ans->m[0][0] = quality;
if (quality < quality_threshold) {
return;
}
SN_ALIGNAS(16) float cross_dot_src[2][4] = { 0 };
DOT_F32_SSE(col0, col1, cross_dot_src[0][0]);
DOT_F32_SSE(col0, col2, cross_dot_src[0][1]);
DOT_F32_SSE(col0, col3, cross_dot_src[0][2]);
DOT_F32_SSE(col1, col2, cross_dot_src[0][3]);
DOT_F32_SSE(col1, col3, cross_dot_src[1][0]);
DOT_F32_SSE(col2, col3, cross_dot_src[1][1]);
register __m128 cross_dot_0 = _mm_load_ps(cross_dot_src[0]);
register __m128 cross_dot_1 = _mm_load_ps(cross_dot_src[1]);
register __m128 base_0 = _mm_shuffle_ps(self_dot, self_dot, _MM_SHUFFLE(1, 0, 0, 0));
register __m128 base_1 = _mm_shuffle_ps(self_dot, self_dot, _MM_SHUFFLE(3, 3, 2, 1));
register __m128 relation_0 =
_mm_div_ps(cross_dot_0, _mm_add_ps(_mm_mul_ps(base_0, base_1), tiny));
base_0 = _mm_shuffle_ps(self_dot, self_dot, _MM_SHUFFLE(2, 3, 2, 1));
base_1 = _mm_shuffle_ps(self_dot, self_dot, _MM_SHUFFLE(3, 3, 3, 3));
register __m128 relation_1 =
_mm_div_ps(cross_dot_0, _mm_add_ps(_mm_mul_ps(base_0, base_1), tiny));
relation_0 = _mm_max_ps(relation_0, relation_1);
FIND_MAX_F32_SSE(relation_0, relation_0);
ans->m[0][1] = _mm_cvtss_f32(relation_0);
}
#endif
@@ -0,0 +1,56 @@
#include "sn_target.h"
#if SN_IS_ARCH(SN_ARCH_X86_64)
#include "sn_intrinsic.h"
#include "sn_intrinsic.context.x86.h"
#include "sn_intrinsic.std.h"
// Matrix transpose implementations
SN_MK_PROC_DECL_STD(float, sse, , transpose)
{
__mk_param_std(RIGHT_ANS, float);
register __m128 col0 = right->x[0];
register __m128 col1 = right->x[1];
register __m128 col2 = right->x[2];
register __m128 col3 = right->x[3];
register __m128 tmp0 = _mm_unpacklo_ps(col0, col1);
register __m128 tmp1 = _mm_unpackhi_ps(col0, col1);
register __m128 tmp2 = _mm_unpacklo_ps(col2, col3);
register __m128 tmp3 = _mm_unpackhi_ps(col2, col3);
ans->x[0] = _mm_movelh_ps(tmp0, tmp2);
ans->x[1] = _mm_movehl_ps(tmp2, tmp0);
ans->x[2] = _mm_movelh_ps(tmp1, tmp3);
ans->x[3] = _mm_movehl_ps(tmp3, tmp1);
}
SN_MK_PROC_DECL_STD(float, avx, , transpose)
{
__mk_param_std(RIGHT_ANS, float);
register __m256 col_low = right->y[0];
register __m256 col_high = right->y[1];
register __m256 unpack_low = _mm256_unpacklo_ps(col_low, col_high);
register __m256 unpack_high = _mm256_unpackhi_ps(col_low, col_high);
col_low = _mm256_permute2f128_ps(unpack_low, unpack_high, 0x20);
col_high = _mm256_permute2f128_ps(unpack_low, unpack_high, 0x31);
ans->y[0] = _mm256_shuffle_ps(col_high, col_low, _MM_SHUFFLE(2, 0, 2, 0));
ans->y[1] = _mm256_shuffle_ps(col_high, col_low, _MM_SHUFFLE(3, 1, 3, 1));
}
SN_MK_PROC_DECL_STD(float, 512, , transpose)
{
__mk_param_std(RIGHT_ANS, float);
register __m512 matrix = right->z[0];
const __m512i transpose_indices =
_mm512_setr_epi32(0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15);
ans->z[0] = _mm512_permutexvar_ps(transpose_indices, matrix);
}
#endif
+288
View File
@@ -0,0 +1,288 @@
#include "sn_target.h"
#if SN_IS_ARCH(SN_ARCH_X86_64)
#include "sn_intrinsic.context.x86.h"
/*
* pack left matrix data to L4 buffer
*
* CONVENSIONS:
*
* buffer_1 is pointer to original colum,
* offset_1 is the offset to the original column,
* count_1 is the count can be packed
*
* C# shceduler is responsible for calculating the offset and count based on the actual buffer size and matrix dimension,
* and make sure that count_1 will not exceeds lenght of the column
*
* scalar pack carries column length and offset data:
* [column length, empty, empty, empty]
*/
SN_MK_PROC_DECL_STD(float, sse, , fma_pack_left)
{
__mk_param_std(RIGHT_ANS_SCALAR, float);
// for SSE branch, no need for rounding, a direct copy is sufficient as long as the buffer is large enough to hold the packed data
memcpy(ans, right + offset_1, sizeof(MAT_KERNEL(float)) * count_2);
}
SN_MK_PROC_DECL_STD(float, avx, , fma_pack_left)
{
__mk_param_std(RIGHT_ANS_SCALAR, float);
size_t i = 0;
for (; i < (count_1 & (~0x1)); i += 2) {
register __m256 ker_0_left = _mm256_load_ps((float *)&right[offset_1 + i].y[0]);
register __m256 ker_0_right = _mm256_load_ps((float *)&right[offset_1 + i].y[1]);
register __m256 ker_1_left = _mm256_load_ps((float *)&right[offset_1 + i].y[0]);
register __m256 ker_1_right = _mm256_load_ps((float *)&right[offset_1 + i].y[1]);
register __m256 col_0 = _mm256_unpacklo_ps(ker_0_left, ker_1_left);
register __m256 col_1 = _mm256_unpackhi_ps(ker_0_left, ker_1_left);
register __m256 col_2 = _mm256_unpacklo_ps(ker_0_right, ker_1_right);
register __m256 col_3 = _mm256_unpackhi_ps(ker_0_right, ker_1_right);
_mm256_store_ps((float *)&ans[i].y[0], col_0);
_mm256_store_ps((float *)&ans[i].y[1], col_1);
_mm256_store_ps((float *)&ans[i].y[2], col_2);
_mm256_store_ps((float *)&ans[i].y[3], col_3);
}
if (count_1 & 0x1) {
register __m256 ker_0_left = _mm256_load_ps((float *)&right[offset_1 + i].y[0]);
register __m256 ker_0_right = _mm256_load_ps((float *)&right[offset_1 + i].y[1]);
register __m256 zero = _mm256_setzero_ps();
register __m256 col_0 = _mm256_unpacklo_ps(ker_0_left, zero);
register __m256 col_1 = _mm256_unpackhi_ps(ker_0_left, zero);
register __m256 col_2 = _mm256_unpacklo_ps(ker_0_right, zero);
register __m256 col_3 = _mm256_unpackhi_ps(ker_0_right, zero);
_mm256_store_ps((float *)&ans[i].y[0], col_0);
_mm256_store_ps((float *)&ans[i].y[1], col_1);
_mm256_store_ps((float *)&ans[i].y[2], col_2);
_mm256_store_ps((float *)&ans[i].y[3], col_3);
}
}
SN_MK_PROC_DECL_STD(float, 512, , fma_pack_left)
{
__mk_param_std(RIGHT_ANS_SCALAR, float);
size_t i = 0;
register __m512 ker_0;
register __m512 ker_1;
register __m512 ker_2;
register __m512 ker_3;
register __m512 col_0;
register __m512 col_1;
register __m512 col_2;
register __m512 col_3;
for (; i < (count_1 & (~0x3)); i += 4) {
ker_0 = _mm512_load_ps((float *)&right[offset_1 + i].z[0]);
ker_1 = _mm512_load_ps((float *)&right[offset_1 + i].z[1]);
ker_2 = _mm512_load_ps((float *)&right[offset_1 + i].z[2]);
ker_3 = _mm512_load_ps((float *)&right[offset_1 + i].z[3]);
col_0 = _mm512_unpacklo_ps(ker_0, ker_2);
col_1 = _mm512_unpackhi_ps(ker_0, ker_2);
col_2 = _mm512_unpacklo_ps(ker_1, ker_3);
col_3 = _mm512_unpackhi_ps(ker_1, ker_3);
_mm512_store_ps((float *)&ans[i].z[0],
_mm512_shuffle_f32x4(col_0, col_2, _MM_SHUFFLE(1, 0, 1, 0)));
_mm512_store_ps((float *)&ans[i].z[1],
_mm512_shuffle_f32x4(col_0, col_2, _MM_SHUFFLE(3, 2, 3, 2)));
_mm512_store_ps((float *)&ans[i].z[2],
_mm512_shuffle_f32x4(col_1, col_3, _MM_SHUFFLE(1, 0, 1, 0)));
_mm512_store_ps((float *)&ans[i].z[3],
_mm512_shuffle_f32x4(col_1, col_3, _MM_SHUFFLE(3, 2, 3, 2)));
}
switch (count_1 & 0x3) {
case 1:
ker_0 = _mm512_load_ps((float *)&right[offset_1 + i].z[0]);
col_0 = _mm512_castps128_ps512(_mm512_extractf32x4_ps(ker_0, 0));
col_1 = _mm512_castps128_ps512(_mm512_extractf32x4_ps(ker_0, 1));
col_2 = _mm512_castps128_ps512(_mm512_extractf32x4_ps(ker_0, 2));
col_3 = _mm512_castps128_ps512(_mm512_extractf32x4_ps(ker_0, 3));
_mm512_store_ps((float *)&ans[i].z[0], col_0);
_mm512_store_ps((float *)&ans[i].z[1], col_1);
_mm512_store_ps((float *)&ans[i].z[2], col_2);
_mm512_store_ps((float *)&ans[i].z[3], col_3);
case 2:
ker_0 = _mm512_load_ps((float *)&right[offset_1 + i].z[0]);
ker_1 = _mm512_load_ps((float *)&right[offset_1 + i].z[1]);
col_0 = _mm512_castps256_ps512(_mm512_extractf32x8_ps(ker_0, 0));
col_1 = _mm512_castps256_ps512(_mm512_extractf32x8_ps(ker_0, 1));
col_2 = _mm512_castps256_ps512(_mm512_extractf32x8_ps(ker_1, 0));
col_3 = _mm512_castps256_ps512(_mm512_extractf32x8_ps(ker_1, 1));
_mm512_store_ps((float *)&ans[i].z[0], col_0);
_mm512_store_ps((float *)&ans[i].z[1], col_1);
_mm512_store_ps((float *)&ans[i].z[2], col_2);
_mm512_store_ps((float *)&ans[i].z[3], col_3);
case 3:
ker_0 = _mm512_load_ps((float *)&right[offset_1 + i].z[0]);
ker_1 = _mm512_load_ps((float *)&right[offset_1 + i].z[1]);
ker_2 = _mm512_load_ps((float *)&right[offset_1 + i].z[2]);
ker_3 = _mm512_setzero_ps();
col_0 = _mm512_unpacklo_ps(ker_0, ker_2);
col_1 = _mm512_unpackhi_ps(ker_0, ker_2);
col_2 = _mm512_unpacklo_ps(ker_1, ker_3);
col_3 = _mm512_unpackhi_ps(ker_1, ker_3);
_mm512_store_ps((float *)&ans[i].z[0],
_mm512_shuffle_f32x4(col_0, col_2, _MM_SHUFFLE(1, 0, 1, 0)));
_mm512_store_ps((float *)&ans[i].z[1],
_mm512_shuffle_f32x4(col_0, col_2, _MM_SHUFFLE(3, 2, 3, 2)));
_mm512_store_ps((float *)&ans[i].z[2],
_mm512_shuffle_f32x4(col_1, col_3, _MM_SHUFFLE(1, 0, 1, 0)));
_mm512_store_ps((float *)&ans[i].z[3],
_mm512_shuffle_f32x4(col_1, col_3, _MM_SHUFFLE(3, 2, 3, 2)));
default:
break;
}
}
/*
* pack right matrix data to L4 buffer
*
* CONVENSIONS:
*
* buffer_1 is pointer to original row,
* offset_1 is the offset to the original row,
* count_1 is the count can be packed
*
* C# shceduler is responsible for calculating the offset and count based on the actual buffer size and matrix dimension,
* and make sure that count_1 will not exceeds lenght of the row
*
* scalar pack carries column length and offset data:
* [column length, empty, empty, empty]
*/
SN_MK_PROC_DECL_STD(float, sse, , fma_pack_right)
{
__mk_param_std(RIGHT_ANS_SCALAR, float);
int col_length = scalar.i32[0];
for (size_t i = 0; i < count_1; i++) {
ans[i] = right[(offset_1 + i) * col_length];
}
}
SN_MK_PROC_DECL_STD(float, avx, , fma_pack_right)
{
__mk_param_std(RIGHT_ANS_SCALAR, float);
int col_length = scalar.i32[0];
size_t i = 0;
for (; i < (count_1 & (~0x1)); i += 2) {
register __m256 ker_0_left =
_mm256_load_ps((float *)&right[(offset_1 + i) * col_length].y[0]);
register __m256 ker_0_right =
_mm256_load_ps((float *)&right[(offset_1 + i) * col_length].y[1]);
register __m256 ker_1_left =
_mm256_load_ps((float *)&right[(offset_1 + i + 1) * col_length].y[0]);
register __m256 ker_1_right =
_mm256_load_ps((float *)&right[(offset_1 + i + 1) * col_length].y[1]);
register __m256 col_0 = _mm256_unpacklo_ps(ker_0_left, ker_1_left);
register __m256 col_1 = _mm256_unpackhi_ps(ker_0_left, ker_1_left);
register __m256 col_2 = _mm256_unpacklo_ps(ker_0_right, ker_1_right);
register __m256 col_3 = _mm256_unpackhi_ps(ker_0_right, ker_1_right);
_mm256_store_ps((float *)&ans[i].y[0], col_0);
_mm256_store_ps((float *)&ans[i].y[1], col_1);
_mm256_store_ps((float *)&ans[i].y[2], col_2);
_mm256_store_ps((float *)&ans[i].y[3], col_3);
}
if (count_1 & 0x1) {
register __m256 ker_0_left =
_mm256_load_ps((float *)&right[(offset_1 + i) * col_length].y[0]);
register __m256 ker_0_right =
_mm256_load_ps((float *)&right[(offset_1 + i) * col_length].y[1]);
register __m256 zero = _mm256_setzero_ps();
register __m256 col_0 = _mm256_unpacklo_ps(ker_0_left, zero);
register __m256 col_1 = _mm256_unpackhi_ps(ker_0_left, zero);
register __m256 col_2 = _mm256_unpacklo_ps(ker_0_right, zero);
register __m256 col_3 = _mm256_unpackhi_ps(ker_0_right, zero);
_mm256_store_ps((float *)&ans[i].y[0], col_0);
_mm256_store_ps((float *)&ans[i].y[1], col_1);
_mm256_store_ps((float *)&ans[i].y[2], col_2);
_mm256_store_ps((float *)&ans[i].y[3], col_3);
}
}
SN_MK_PROC_DECL_STD(float, 512, , fma_pack_right)
{
__mk_param_std(RIGHT_ANS_SCALAR, float);
int col_length = scalar.i32[0];
size_t i = 0;
register __m512 ker_0;
register __m512 ker_1;
register __m512 ker_2;
register __m512 ker_3;
register __m512 col_0;
register __m512 col_1;
register __m512 col_2;
register __m512 col_3;
for (; i < (count_1 & (~0x3)); i += 4) {
ker_0 = _mm512_load_ps((float *)&right[(offset_1 + i + 0) * col_length].z[0]);
ker_1 = _mm512_load_ps((float *)&right[(offset_1 + i + 1) * col_length].z[0]);
ker_2 = _mm512_load_ps((float *)&right[(offset_1 + i + 2) * col_length].z[0]);
ker_3 = _mm512_load_ps((float *)&right[(offset_1 + i + 3) * col_length].z[0]);
col_0 = _mm512_unpacklo_ps(ker_0, ker_2);
col_1 = _mm512_unpackhi_ps(ker_0, ker_2);
col_2 = _mm512_unpacklo_ps(ker_1, ker_3);
col_3 = _mm512_unpackhi_ps(ker_1, ker_3);
_mm512_store_ps((float *)&ans[i].z[0],
_mm512_shuffle_f32x4(col_0, col_2, _MM_SHUFFLE(1, 0, 1, 0)));
_mm512_store_ps((float *)&ans[i].z[1],
_mm512_shuffle_f32x4(col_0, col_2, _MM_SHUFFLE(3, 2, 3, 2)));
_mm512_store_ps((float *)&ans[i].z[2],
_mm512_shuffle_f32x4(col_1, col_3, _MM_SHUFFLE(1, 0, 1, 0)));
_mm512_store_ps((float *)&ans[i].z[3],
_mm512_shuffle_f32x4(col_1, col_3, _MM_SHUFFLE(3, 2, 3, 2)));
}
switch (count_1 & 0x3) {
case 1:
ker_0 = _mm512_load_ps((float *)&right[(offset_1 + i) * col_length].z[0]);
col_0 = _mm512_castps128_ps512(_mm512_extractf32x4_ps(ker_0, 0));
col_1 = _mm512_castps128_ps512(_mm512_extractf32x4_ps(ker_0, 1));
col_2 = _mm512_castps128_ps512(_mm512_extractf32x4_ps(ker_0, 2));
col_3 = _mm512_castps128_ps512(_mm512_extractf32x4_ps(ker_0, 3));
_mm512_store_ps((float *)&ans[i].z[0], col_0);
_mm512_store_ps((float *)&ans[i].z[1], col_1);
_mm512_store_ps((float *)&ans[i].z[2], col_2);
_mm512_store_ps((float *)&ans[i].z[3], col_3);
case 2:
ker_0 = _mm512_load_ps((float *)&right[(offset_1 + i + 0) * col_length].z[0]);
ker_1 = _mm512_load_ps((float *)&right[(offset_1 + i + 1) * col_length].z[1]);
col_0 = _mm512_castps256_ps512(_mm512_extractf32x8_ps(ker_0, 0));
col_1 = _mm512_castps256_ps512(_mm512_extractf32x8_ps(ker_0, 1));
col_2 = _mm512_castps256_ps512(_mm512_extractf32x8_ps(ker_1, 0));
col_3 = _mm512_castps256_ps512(_mm512_extractf32x8_ps(ker_1, 1));
_mm512_store_ps((float *)&ans[i].z[0], col_0);
_mm512_store_ps((float *)&ans[i].z[1], col_1);
_mm512_store_ps((float *)&ans[i].z[2], col_2);
_mm512_store_ps((float *)&ans[i].z[3], col_3);
case 3:
ker_0 = _mm512_load_ps((float *)&right[(offset_1 + i + 0) * col_length].z[0]);
ker_1 = _mm512_load_ps((float *)&right[(offset_1 + i + 1) * col_length].z[1]);
ker_2 = _mm512_load_ps((float *)&right[(offset_1 + i + 2) * col_length].z[2]);
ker_3 = _mm512_setzero_ps();
col_0 = _mm512_unpacklo_ps(ker_0, ker_2);
col_1 = _mm512_unpackhi_ps(ker_0, ker_2);
col_2 = _mm512_unpacklo_ps(ker_1, ker_3);
col_3 = _mm512_unpackhi_ps(ker_1, ker_3);
_mm512_store_ps((float *)&ans[i].z[0],
_mm512_shuffle_f32x4(col_0, col_2, _MM_SHUFFLE(1, 0, 1, 0)));
_mm512_store_ps((float *)&ans[i].z[1],
_mm512_shuffle_f32x4(col_0, col_2, _MM_SHUFFLE(3, 2, 3, 2)));
_mm512_store_ps((float *)&ans[i].z[2],
_mm512_shuffle_f32x4(col_1, col_3, _MM_SHUFFLE(1, 0, 1, 0)));
_mm512_store_ps((float *)&ans[i].z[3],
_mm512_shuffle_f32x4(col_1, col_3, _MM_SHUFFLE(3, 2, 3, 2)));
default:
break;
}
}
#endif
+375
View File
@@ -0,0 +1,375 @@
#include "sn_target.h"
#include <immintrin.h>
#if SN_IS_ARCH(SN_ARCH_X86_64)
// #include "sn_intrinsic.h"
#include "sn_template.fma.x86.h"
#include "sn_intrinsic.std.h"
#include "sn_intrinsic.context.x86.h"
#include <stdio.h>
#define _FMA_SELECT(i) _MM_SHUFFLE(i, i, i, i)
SN_MK_PROC_DECL_STD(float, sse, , fma)
{
__mk_param_std(LEFT_RIGHT_ANS, float);
SN_MAT_KERNEL_LOCAL *restrict ans_cur = ans + offset_2;
register __m128 c0 = _mm_load_ps((float *)&ans_cur->x[0]);
register __m128 c1 = _mm_load_ps((float *)&ans_cur->x[1]);
register __m128 c2 = _mm_load_ps((float *)&ans_cur->x[2]);
register __m128 c3 = _mm_load_ps((float *)&ans_cur->x[3]);
register __m128 bcol;
for (size_t z = 0; z < count_0; z++) {
SN_MAT_KERNEL_LOCAL const *right_base = right;
SN_MAT_KERNEL_LOCAL const *left_base = left;
const __m128 a0 = _mm_load_ps((float *)&left_base->x[0]);
const __m128 a1 = _mm_load_ps((float *)&left_base->x[1]);
const __m128 a2 = _mm_load_ps((float *)&left_base->x[2]);
const __m128 a3 = _mm_load_ps((float *)&left_base->x[3]);
/* Column 0 */
bcol = _mm_load_ps((float *)&right_base->x[0]);
c0 = _mm_add_ps(c0, _mm_mul_ps(a0, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(0))));
c0 = _mm_add_ps(c0, _mm_mul_ps(a1, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(1))));
c0 = _mm_add_ps(c0, _mm_mul_ps(a2, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(2))));
c0 = _mm_add_ps(c0, _mm_mul_ps(a3, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(3))));
/* Column 1 */
bcol = _mm_load_ps((float *)&right_base->x[1]);
c1 = _mm_add_ps(c1, _mm_mul_ps(a0, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(0))));
c1 = _mm_add_ps(c1, _mm_mul_ps(a1, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(1))));
c1 = _mm_add_ps(c1, _mm_mul_ps(a2, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(2))));
c1 = _mm_add_ps(c1, _mm_mul_ps(a3, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(3))));
/* Column 2 */
bcol = _mm_load_ps((float *)&right_base->x[2]);
c2 = _mm_add_ps(c2, _mm_mul_ps(a0, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(0))));
c2 = _mm_add_ps(c2, _mm_mul_ps(a1, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(1))));
c2 = _mm_add_ps(c2, _mm_mul_ps(a2, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(2))));
c2 = _mm_add_ps(c2, _mm_mul_ps(a3, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(3))));
/* Column 3 */
bcol = _mm_load_ps((float *)&right_base->x[3]);
c3 = _mm_add_ps(c3, _mm_mul_ps(a0, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(0))));
c3 = _mm_add_ps(c3, _mm_mul_ps(a1, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(1))));
c3 = _mm_add_ps(c3, _mm_mul_ps(a2, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(2))));
c3 = _mm_add_ps(c3, _mm_mul_ps(a3, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(3))));
}
_mm_store_ps((float *)&ans_cur->x[0], c0);
_mm_store_ps((float *)&ans_cur->x[1], c1);
_mm_store_ps((float *)&ans_cur->x[2], c2);
_mm_store_ps((float *)&ans_cur->x[3], c3);
}
SN_MK_PROC_DECL_STD(float, avx, , fma)
{
__mk_param_std(LEFT_RIGHT_ANS, float);
SN_MAT_KERNEL_LOCAL *restrict ans_cur = ans + offset_2;
register __m256 c0 = _mm256_load_ps((float *)&ans_cur->y[0]);
register __m256 c1 = _mm256_load_ps((float *)&ans_cur->y[1]);
register __m256 c2 = _mm256_load_ps((float *)&ans_cur->y[2]);
register __m256 c3 = _mm256_load_ps((float *)&ans_cur->y[3]);
for (size_t z = 0; z < count_0; z++) {
SN_MAT_KERNEL_LOCAL const *b_base = right;
SN_MAT_KERNEL_LOCAL const *a_base = left;
register __m256 b0 = _mm256_load_ps((float *)&b_base->y[0]);
register __m256 b1 = _mm256_load_ps((float *)&b_base->y[1]);
register __m256 b2 = _mm256_load_ps((float *)&b_base->y[2]);
register __m256 b3 = _mm256_load_ps((float *)&b_base->y[3]);
register __m256 a0 = _mm256_load_ps((float *)&a_base->y[0]);
register __m256 a1 = _mm256_load_ps((float *)&a_base->y[1]);
register __m256 a2 = _mm256_load_ps((float *)&a_base->y[2]);
register __m256 a3 = _mm256_load_ps((float *)&a_base->y[3]);
SN_MK_FMA_F32_AVX2_CYCLE(0);
SN_MK_FMA_F32_AVX2_CYCLE(1);
SN_MK_FMA_F32_AVX2_CYCLE(2);
SN_MK_FMA_F32_AVX2_CYCLE(3);
}
_mm256_store_ps((float *)&(ans_cur->y[0]), c0);
_mm256_store_ps((float *)&(ans_cur->y[1]), c1);
_mm256_store_ps((float *)&(ans_cur->y[2]), c2);
_mm256_store_ps((float *)&(ans_cur->y[3]), c3);
}
SN_MK_PROC_DECL_STD(float, avx, _fma, fma)
{
__mk_param_std(LEFT_RIGHT_ANS, float);
SN_MAT_KERNEL_LOCAL *restrict ans_cur = ans + offset_2;
register __m256 c0 = _mm256_load_ps((float *)&ans_cur->y[0]);
register __m256 c1 = _mm256_load_ps((float *)&ans_cur->y[1]);
register __m256 c2 = _mm256_load_ps((float *)&ans_cur->y[2]);
register __m256 c3 = _mm256_load_ps((float *)&ans_cur->y[3]);
for (size_t z = 0; z < count_0; z += 2) {
const SN_MAT_KERNEL_LOCAL *a_base = left + z;
const SN_MAT_KERNEL_LOCAL *b_base = right + z;
register __m256 b0 = _mm256_load_ps((float *)&b_base->y[0]);
register __m256 b1 = _mm256_load_ps((float *)&b_base->y[1]);
register __m256 b2 = _mm256_load_ps((float *)&b_base->y[2]);
register __m256 b3 = _mm256_load_ps((float *)&b_base->y[3]);
register __m256 a0 = _mm256_load_ps((float *)&a_base->y[0]);
register __m256 a1 = _mm256_load_ps((float *)&a_base->y[1]);
register __m256 a2 = _mm256_load_ps((float *)&a_base->y[2]);
register __m256 a3 = _mm256_load_ps((float *)&a_base->y[3]);
SN_MK_FMA_F32_AVX2_CYCLE(0);
SN_MK_FMA_F32_AVX2_CYCLE(1);
SN_MK_FMA_F32_AVX2_CYCLE(2);
SN_MK_FMA_F32_AVX2_CYCLE(3);
}
_mm256_store_ps((float *)&(ans_cur->y[0]), c0);
_mm256_store_ps((float *)&(ans_cur->y[1]), c1);
_mm256_store_ps((float *)&(ans_cur->y[2]), c2);
_mm256_store_ps((float *)&(ans_cur->y[3]), c3);
}
SN_MK_PROC_DECL_STD(float, 512, , fma)
{
__mk_param_std(LEFT_RIGHT_ANS, float);
SN_MAT_KERNEL_LOCAL *restrict ans_cur = ans + offset_2;
register __m512 c0 = _mm512_load_ps((float *)&ans_cur->z[0]);
register __m512 c1 = _mm512_load_ps((float *)&ans_cur->z[0]);
register __m512 c2 = _mm512_load_ps((float *)&ans_cur->z[0]);
register __m512 c3 = _mm512_load_ps((float *)&ans_cur->z[0]);
for (size_t z = 0; z < count_0; z += 4) {
const SN_MAT_KERNEL_LOCAL *a_base = left + z;
const SN_MAT_KERNEL_LOCAL *b_base = right + z;
register __m512 b0 = _mm512_load_ps((float *)&b_base->z[0]);
register __m512 b1 = _mm512_load_ps((float *)&b_base->z[1]);
register __m512 b2 = _mm512_load_ps((float *)&b_base->z[2]);
register __m512 b3 = _mm512_load_ps((float *)&b_base->z[3]);
register __m512 a0 = _mm512_load_ps((float *)&a_base->z[0]);
register __m512 a1 = _mm512_load_ps((float *)&a_base->z[1]);
register __m512 a2 = _mm512_load_ps((float *)&a_base->z[2]);
register __m512 a3 = _mm512_load_ps((float *)&a_base->z[3]);
SN_MK_FMA_F32_512_CYCLE(0);
SN_MK_FMA_F32_512_CYCLE(1);
SN_MK_FMA_F32_512_CYCLE(2);
SN_MK_FMA_F32_512_CYCLE(3);
}
_mm512_store_ps((float *)&ans_cur->z[0], c0);
_mm512_store_ps((float *)&ans_cur->z[1], c1);
_mm512_store_ps((float *)&ans_cur->z[2], c2);
_mm512_store_ps((float *)&ans_cur->z[3], c3);
}
SN_MK_PROC_DECL_STD(double, sse, , fma)
{
__mk_param_std(LEFT_RIGHT_ANS, double);
SN_MAT_KERNEL_LOCAL *restrict ans_cur = ans + offset_2;
register __m128d c0_lo = ans_cur->x[0];
register __m128d c0_hi = ans_cur->x[1];
register __m128d c1_lo = ans_cur->x[2];
register __m128d c1_hi = ans_cur->x[3];
register __m128d c2_lo = ans_cur->x[4];
register __m128d c2_hi = ans_cur->x[5];
register __m128d c3_lo = ans_cur->x[6];
register __m128d c3_hi = ans_cur->x[7];
for (size_t z = 0; z < count_0; z++) {
SN_MAT_KERNEL_LOCAL const *right_base = right;
SN_MAT_KERNEL_LOCAL const *left_base = left;
const double *right_values = &right_base->m[0][0];
const double *left_values = &left_base->m[0][0];
SN_MK_FMA_F64_SSE_PHASE(0);
SN_MK_FMA_F64_SSE_PHASE(1);
SN_MK_FMA_F64_SSE_PHASE(2);
SN_MK_FMA_F64_SSE_PHASE(3);
}
ans_cur->x[0] = c0_lo;
ans_cur->x[1] = c0_hi;
ans_cur->x[2] = c1_lo;
ans_cur->x[3] = c1_hi;
ans_cur->x[4] = c2_lo;
ans_cur->x[5] = c2_hi;
ans_cur->x[6] = c3_lo;
ans_cur->x[7] = c3_hi;
}
SN_MK_PROC_DECL_STD(double, avx, , fma)
{
__mk_param_std(LEFT_RIGHT_ANS, double);
SN_MAT_KERNEL_LOCAL *restrict ans_cur = ans + offset_2;
register __m256d c0 = _mm256_load_pd((double *)&ans_cur->y[0]);
register __m256d c1 = _mm256_load_pd((double *)&ans_cur->y[1]);
register __m256d c2 = _mm256_load_pd((double *)&ans_cur->y[2]);
register __m256d c3 = _mm256_load_pd((double *)&ans_cur->y[3]);
register __m256d bcol;
for (size_t z = 0; z < count_0; z++) {
SN_MAT_KERNEL_LOCAL const *right_base = right;
SN_MAT_KERNEL_LOCAL const *left_base = left;
const __m256d a0 = _mm256_load_pd((double *)&left_base->y[0]);
const __m256d a1 = _mm256_load_pd((double *)&left_base->y[1]);
const __m256d a2 = _mm256_load_pd((double *)&left_base->y[2]);
const __m256d a3 = _mm256_load_pd((double *)&left_base->y[3]);
/* Column 0 */
bcol = _mm256_load_pd((double *)&right_base->y[0]);
c0 = _mm256_add_pd(c0,
_mm256_mul_pd(a0, _mm256_permute4x64_pd(bcol, _FMA_SELECT(0))));
c0 = _mm256_add_pd(c0,
_mm256_mul_pd(a1, _mm256_permute4x64_pd(bcol, _FMA_SELECT(1))));
c0 = _mm256_add_pd(c0,
_mm256_mul_pd(a2, _mm256_permute4x64_pd(bcol, _FMA_SELECT(2))));
c0 = _mm256_add_pd(c0,
_mm256_mul_pd(a3, _mm256_permute4x64_pd(bcol, _FMA_SELECT(3))));
/* Column 1 */
bcol = _mm256_load_pd((double *)&right_base->y[1]);
c1 = _mm256_add_pd(c1,
_mm256_mul_pd(a0, _mm256_permute4x64_pd(bcol, _FMA_SELECT(0))));
c1 = _mm256_add_pd(c1,
_mm256_mul_pd(a1, _mm256_permute4x64_pd(bcol, _FMA_SELECT(1))));
c1 = _mm256_add_pd(c1,
_mm256_mul_pd(a2, _mm256_permute4x64_pd(bcol, _FMA_SELECT(2))));
c1 = _mm256_add_pd(c1,
_mm256_mul_pd(a3, _mm256_permute4x64_pd(bcol, _FMA_SELECT(3))));
/* Column 2 */
bcol = _mm256_load_pd((double *)&right_base->y[2]);
c2 = _mm256_add_pd(c2,
_mm256_mul_pd(a0, _mm256_permute4x64_pd(bcol, _FMA_SELECT(0))));
c2 = _mm256_add_pd(c2,
_mm256_mul_pd(a1, _mm256_permute4x64_pd(bcol, _FMA_SELECT(1))));
c2 = _mm256_add_pd(c2,
_mm256_mul_pd(a2, _mm256_permute4x64_pd(bcol, _FMA_SELECT(2))));
c2 = _mm256_add_pd(c2,
_mm256_mul_pd(a3, _mm256_permute4x64_pd(bcol, _FMA_SELECT(3))));
/* Column 3 */
bcol = _mm256_load_pd((double *)&right_base->y[3]);
c3 = _mm256_add_pd(c3,
_mm256_mul_pd(a0, _mm256_permute4x64_pd(bcol, _FMA_SELECT(0))));
c3 = _mm256_add_pd(c3,
_mm256_mul_pd(a1, _mm256_permute4x64_pd(bcol, _FMA_SELECT(1))));
c3 = _mm256_add_pd(c3,
_mm256_mul_pd(a2, _mm256_permute4x64_pd(bcol, _FMA_SELECT(2))));
c3 = _mm256_add_pd(c3,
_mm256_mul_pd(a3, _mm256_permute4x64_pd(bcol, _FMA_SELECT(3))));
}
_mm256_store_pd((double *)&ans_cur->y[0], c0);
_mm256_store_pd((double *)&ans_cur->y[1], c1);
_mm256_store_pd((double *)&ans_cur->y[2], c2);
_mm256_store_pd((double *)&ans_cur->y[3], c3);
}
SN_MK_PROC_DECL_STD(double, avx, _fma, fma)
{
__mk_param_std(LEFT_RIGHT_ANS, double);
SN_MAT_KERNEL_LOCAL *restrict ans_cur = ans + offset_2;
register __m256d c0 = _mm256_load_pd((double *)&ans_cur->y[0]);
register __m256d c1 = _mm256_load_pd((double *)&ans_cur->y[1]);
register __m256d c2 = _mm256_load_pd((double *)&ans_cur->y[2]);
register __m256d c3 = _mm256_load_pd((double *)&ans_cur->y[3]);
register __m256d bcol;
for (size_t z = 0; z < count_0; z++) {
SN_MAT_KERNEL_LOCAL const *right_base = right;
SN_MAT_KERNEL_LOCAL const *left_base = left;
const __m256d a0 = _mm256_load_pd((double *)&left_base->y[0]);
const __m256d a1 = _mm256_load_pd((double *)&left_base->y[1]);
const __m256d a2 = _mm256_load_pd((double *)&left_base->y[2]);
const __m256d a3 = _mm256_load_pd((double *)&left_base->y[3]);
/* Column 0 */
bcol = _mm256_load_pd((double *)&right_base->y[0]);
c0 = _mm256_fmadd_pd(a0, _mm256_permute4x64_pd(bcol, _FMA_SELECT(0)), c0);
c0 = _mm256_fmadd_pd(a1, _mm256_permute4x64_pd(bcol, _FMA_SELECT(1)), c0);
c0 = _mm256_fmadd_pd(a2, _mm256_permute4x64_pd(bcol, _FMA_SELECT(2)), c0);
c0 = _mm256_fmadd_pd(a3, _mm256_permute4x64_pd(bcol, _FMA_SELECT(3)), c0);
/* Column 1 */
bcol = _mm256_load_pd((double *)&right_base->y[1]);
c1 = _mm256_fmadd_pd(a0, _mm256_permute4x64_pd(bcol, _FMA_SELECT(0)), c1);
c1 = _mm256_fmadd_pd(a1, _mm256_permute4x64_pd(bcol, _FMA_SELECT(1)), c1);
c1 = _mm256_fmadd_pd(a2, _mm256_permute4x64_pd(bcol, _FMA_SELECT(2)), c1);
c1 = _mm256_fmadd_pd(a3, _mm256_permute4x64_pd(bcol, _FMA_SELECT(3)), c1);
/* Column 2 */
bcol = _mm256_load_pd((double *)&right_base->y[2]);
c2 = _mm256_fmadd_pd(a0, _mm256_permute4x64_pd(bcol, _FMA_SELECT(0)), c2);
c2 = _mm256_fmadd_pd(a1, _mm256_permute4x64_pd(bcol, _FMA_SELECT(1)), c2);
c2 = _mm256_fmadd_pd(a2, _mm256_permute4x64_pd(bcol, _FMA_SELECT(2)), c2);
c2 = _mm256_fmadd_pd(a3, _mm256_permute4x64_pd(bcol, _FMA_SELECT(3)), c2);
/* Column 3 */
bcol = _mm256_load_pd((double *)&right_base->y[3]);
c3 = _mm256_fmadd_pd(a0, _mm256_permute4x64_pd(bcol, _FMA_SELECT(0)), c3);
c3 = _mm256_fmadd_pd(a1, _mm256_permute4x64_pd(bcol, _FMA_SELECT(1)), c3);
c3 = _mm256_fmadd_pd(a2, _mm256_permute4x64_pd(bcol, _FMA_SELECT(2)), c3);
c3 = _mm256_fmadd_pd(a3, _mm256_permute4x64_pd(bcol, _FMA_SELECT(3)), c3);
}
_mm256_store_pd((double *)&ans_cur->y[0], c0);
_mm256_store_pd((double *)&ans_cur->y[1], c1);
_mm256_store_pd((double *)&ans_cur->y[2], c2);
_mm256_store_pd((double *)&ans_cur->y[3], c3);
}
SN_MK_PROC_DECL_STD(double, 512, , fma)
{
__mk_param_std(LEFT_RIGHT_ANS, double);
SN_MAT_KERNEL_LOCAL *restrict ans_cur = ans + offset_2;
register __m512d c0 = _mm512_load_pd((double *)&ans_cur->z[0]);
register __m512d c1 = _mm512_load_pd((double *)&ans_cur->z[1]);
register __m512d c2 = _mm512_load_pd((double *)&ans_cur->z[2]);
register __m512d c3 = _mm512_load_pd((double *)&ans_cur->z[3]);
for (size_t z = 0; z < count_0; z += 2) {
SN_MAT_KERNEL_LOCAL const *right_base = right + z;
SN_MAT_KERNEL_LOCAL const *left_base = left + z;
register __m512d a0 = _mm512_load_pd((double *)&left_base->z[0]);
register __m512d a1 = _mm512_load_pd((double *)&left_base->z[1]);
register __m512d a2 = _mm512_load_pd((double *)&left_base->z[2]);
register __m512d a3 = _mm512_load_pd((double *)&left_base->z[3]);
register __m512d b0 = _mm512_load_pd((double *)&right_base->z[0]);
register __m512d b1 = _mm512_load_pd((double *)&right_base->z[1]);
register __m512d b2 = _mm512_load_pd((double *)&right_base->z[2]);
register __m512d b3 = _mm512_load_pd((double *)&right_base->z[3]);
SN_MK_FMA_F64_AVX2_CYCLE(0);
SN_MK_FMA_F64_AVX2_CYCLE(1);
SN_MK_FMA_F64_AVX2_CYCLE(2);
SN_MK_FMA_F64_AVX2_CYCLE(3);
}
_mm512_store_pd((double *)&ans_cur->z[0], c0);
_mm512_store_pd((double *)&ans_cur->z[1], c1);
_mm512_store_pd((double *)&ans_cur->z[2], c2);
_mm512_store_pd((double *)&ans_cur->z[3], c3);
}
#endif
+50
View File
@@ -0,0 +1,50 @@
#include "sn_target.h"
#if SN_IS_ARCH(SN_ARCH_X86_64)
#include "sn_internal.h"
#include "sn_intrinsic.context.x86.h"
#include "sn_intrinsic.std.h"
#include "sn_intrinsic.matkernel.h"
#include <emmintrin.h>
#include <immintrin.h>
#include <pmmintrin.h>
SN_MK_PROC_DECL_STD(float, sse, , k_quality)
{
__mk_param_std(RIGHT_ANS_SCALAR, float);
}
INTERNAL SN_STD_TARGET_ATTR(sse, ) void kp_check_f32(mat_kernel *kernel)
{
typedef MAT_KERNEL(float) SN_MAT;
SN_MAT const *k = (SN_MAT const *)kernel;
float cross[4];
cross[0] = k->m[0][0];
cross[1] = k->m[1][1];
cross[2] = k->m[2][2];
cross[3] = k->m[3][3];
float evaluation[4];
register __m128 cross_vec = _mm_loadu_ps((float *)cross);
register __m128 abs_mask = _mm_castsi128_ps(_mm_set1_epi32(0x7fffffff));
cross_vec = _mm_and_ps(cross_vec, abs_mask);
register __m128 average_vec = _mm_hadd_ps(cross_vec, cross_vec);
average_vec = _mm_hadd_ps(average_vec, average_vec);
register __m128 quarter_vec = _mm_set1_ps(0.25f);
average_vec = _mm_mul_ps(average_vec, quarter_vec);
register __m128 stddev_vec = _mm_sub_ps(cross_vec, average_vec);
stddev_vec = _mm_mul_ps(stddev_vec, stddev_vec);
stddev_vec = _mm_hadd_ps(stddev_vec, stddev_vec);
stddev_vec = _mm_hadd_ps(stddev_vec, stddev_vec);
stddev_vec = _mm_mul_ps(stddev_vec, quarter_vec);
stddev_vec = _mm_sqrt_ps(stddev_vec);
evaluation[0] = _mm_cvtss_f32(average_vec);
evaluation[1] = _mm_cvtss_f32(stddev_vec);
}
#endif
@@ -0,0 +1,33 @@
#include "sn_target.h"
#if SN_IS_ARCH(SN_ARCH_X86_64)
#include "sn_internal.h"
#include "sn_intrinsic.h"
static __m128i facIncrement;
static __m128i facBase;
INTERNAL uint64_t SN_DECL factorial_simd_sse(int n)
{
facIncrement = _mm_set1_epi32(4);
facBase = _mm_setr_epi32(1, 2, 3, 4);
__m128i baseResult = facBase;
__m128i facBaseLocal = facBase;
const int cycle = (n - 3) >> 2;
const uint32_t rest = (cycle << 2) + 1;
for (size_t i = 0; i < cycle; i++) {
facBaseLocal = _mm_add_epi32(facBaseLocal, facIncrement);
baseResult = _mm_mullo_epi32(baseResult, facBaseLocal);
}
SN_ALIGNAS(16) int32_t lanes[4];
_mm_store_si128((__m128i *)lanes, baseResult);
uint64_t result = (uint64_t)lanes[0] * (uint64_t)lanes[1] * (uint64_t)lanes[2] *
(uint64_t)lanes[3];
for (uint64_t i = rest; i < n; i++) {
result *= i;
}
return result;
}
#endif
+50
View File
@@ -0,0 +1,50 @@
#include "sn_target.h"
#if SN_IS_ARCH(SN_ARCH_X86_64)
#include "StgSharpNative.h"
INTERNAL int SN_DECL city_hash_simplify_sse(byte const *str, int const length)
{
uint32_t hash = length, size = length << 1;
byte const *end = str + size;
byte const *ptr = str;
if (size < 4) {
for (; ptr < end; ptr += 2) {
hash = _mm_crc32_u16(hash, *ptr);
}
} else if (size < 16) {
for (; ptr < end; ptr += 4) {
hash = _mm_crc32_u32(hash, ((uint32_t const *)ptr)[0]);
}
for (; ptr < end; ptr++) {
hash = _mm_crc32_u16(hash, *ptr);
}
} else {
byte const *last_128 = end - 16;
byte const *last_32 = end - 4;
uint64_t hash64 = size;
for (; ptr < last_128; ptr += 16) {
__m128i vec = _mm_loadu_si128((__m128i const *)ptr);
hash64 = _mm_crc32_u64(hash64, _mm_extract_epi64(vec, 0));
hash64 = _mm_crc32_u64(hash64, _mm_extract_epi64(vec, 1));
}
hash = _mm_crc32_u8(hash, hash64);
hash = _mm_crc32_u8(hash, hash64 >> 32);
for (; ptr < last_32; ptr += 4) {
hash = _mm_crc32_u32(hash, ((uint32_t const *)ptr)[0]);
}
for (; ptr < end; ptr += 2) {
hash = _mm_crc32_u8(hash, *ptr);
}
}
hash ^= hash >> 16;
hash *= 0x85ebca6b;
hash ^= hash >> 13;
hash *= 0xc2b2ae35;
hash ^= hash >> 16;
return hash;
}
#endif
+329
View File
@@ -0,0 +1,329 @@
#include "sn_target.h"
/* clang-format off */
#if SN_IS_ARCH(SN_ARCH_X86_64)
#include "sn_intrinsic.h"
#include "StgSharpNative.h"
static SIMDID global_simdid = { 0 };
#include "sn_intrinsic.context.x86.h"
#include <immintrin.h>
#include <string.h>
#if defined(_MSC_VER)
#include <intrin.h>
#else
#include <cpuid.h>
#endif
static void get_cpuid(int leaf, int subleaf, int *eax, int *ebx, int *ecx, int *edx)
{
#if defined(_MSC_VER)
int regs[4];
__cpuidex(regs, leaf, subleaf);
*eax = regs[0];
*ebx = regs[1];
*ecx = regs[2];
*edx = regs[3];
#else
unsigned int a, b, c, d;
__cpuid_count((unsigned int)leaf, (unsigned int)subleaf, a, b, c, d);
*eax = (int)a;
*ebx = (int)b;
*ecx = (int)c;
*edx = (int)d;
#endif
}
/* XCR0 is per logical processor. Reading it after thread affinity is applied
* verifies that the OS saves/restores the register state needed by this core. */
static unsigned long long read_xcr0(void)
{
#if defined(_MSC_VER)
return _xgetbv(0);
#else
return __builtin_ia32_xgetbv(0);
#endif
}
static int os_has_xsave_state(unsigned long long required)
{
return (read_xcr0() & required) == required;
}
#define XCR0_X87 (1ull << 0)
#define XCR0_XMM (1ull << 1)
#define XCR0_YMM (1ull << 2)
#define XCR0_OPMASK (1ull << 5)
#define XCR0_ZMM_HI256 (1ull << 6)
#define XCR0_HI16_ZMM (1ull << 7)
#define XCR0_AVX_STATE (XCR0_X87 | XCR0_XMM | XCR0_YMM)
#define XCR0_AVX512_STATE \
(XCR0_AVX_STATE | XCR0_OPMASK | XCR0_ZMM_HI256 | XCR0_HI16_ZMM)
static void decode_family_model(int cpuid1_eax, int *family, int *model)
{
int family_value = 0;
int model_value = 0;
const int base_fam = (cpuid1_eax >> 8) & 0xF;
const int ext_fam = (cpuid1_eax >> 20) & 0xFF;
const int base_model = (cpuid1_eax >> 4) & 0xF;
const int ext_model = (cpuid1_eax >> 16) & 0xF;
if (base_fam == 0xF) {
family_value = base_fam + ext_fam;
} else {
family_value = base_fam;
}
if ((base_fam == 0x6) || (base_fam == 0xF)) {
model_value = (ext_model << 4) | base_model;
} else {
model_value = base_model;
}
*family = family_value;
*model = model_value;
}
/* AVX-512 execution-width classification for AMD.
* Family 19h (Zen4) uses a 256-bit dual-pumped path.
* Family 1Ah models 00h-0Fh are documented for the full-width EPYC 9005
* implementation. Models 10h-1Fh are intentionally UNKNOWN: that CPUID
* range is used by both full-width EPYC 9005 Zen5c and dual-pumped EPYC
* 8005 products, and CPUID has no architectural execution-width bit. */
static uint64_t amd_avx512_impl(int family, int model)
{
if (family == 0x19) {
return SIMDID_AVX512_IMPL_DUAL;
}
if (family == 0x1A) {
if (model >= 0x00 && model <= 0x0F) {
return SIMDID_AVX512_IMPL_NATIVE;
}
}
return SIMDID_AVX512_IMPL_UNKNOWN;
}
/* Returns 1 if this AMD/Hygon CPU uses split-lane YMM (two fused 128-bit execution units).
* - AMD Zen1 (family 0x17, model < 0x30) and Zen+ (0x18-0x2F) use a split/glued path.
* - Hygon Dhyana is architecturally Zen1-based; always split.
* - AMD Zen2+ (family 0x17 model >= 0x30, family 0x19+) have native 256-bit execution. */
static int is_amd_glue_ymm(const char *vendor, int cpuid1_eax)
{
int family = 0;
int model = 0;
if (strcmp(vendor, "HygonGenuine") == 0) {
return 1;
}
if (strcmp(vendor, "AuthenticAMD") != 0) {
return 0;
}
decode_family_model(cpuid1_eax, &family, &model);
return (family == 0x17) && (model < 0x30);
}
SN_API SIMDID SN_DECL sn_get_simd_level_local()
{
SIMDID simdid = { 0 };
int eax = 0;
int ebx = 0;
int ecx = 0;
int edx = 0;
int family = 0;
int model = 0;
get_cpuid(0, 0, &eax, &ebx, &ecx, &edx);
const int max_basic_leaf = eax;
char vendor[13];
memcpy(vendor + 0, &ebx, 4);
memcpy(vendor + 4, &edx, 4);
memcpy(vendor + 8, &ecx, 4);
vendor[12] = '\0';
const int is_intel = (strcmp(vendor, "GenuineIntel") == 0);
const int is_amd = (strcmp(vendor, "AuthenticAMD") == 0);
const int is_hygon = (strcmp(vendor, "HygonGenuine") == 0);
const int is_centaur = (strcmp(vendor, "CentaurHauls") == 0);
const int is_zhaoxin = (strcmp(vendor, " Shanghai ") == 0);
const int is_via = is_centaur || is_zhaoxin;
get_cpuid(1, 0, &eax, &ebx, &ecx, &edx);
const int cpuid1_eax = eax;
decode_family_model(cpuid1_eax, &family, &model);
(void)model;
const int has_sse42 = (ecx & (1 << 20)) != 0;
const int has_avx = (ecx & (1 << 28)) != 0;
const int has_osxsave = (ecx & (1 << 27)) != 0;
const int has_fma = (ecx & (1 << 12)) != 0;
const int has_f16c = (ecx & (1 << 29)) != 0;
const uint64_t root = SIMDID_PACK_ROOT(SIMDID_ROOT_X86_64);
uint64_t manu_code = SIMDID_MANU_UNKNOWN;
if (is_intel) {
manu_code = SIMDID_MANU_INTEL;
} else if (is_amd) {
manu_code = SIMDID_MANU_AMD;
} else if (is_hygon) {
manu_code = SIMDID_MANU_HYGON;
} else if (is_via) {
manu_code = SIMDID_MANU_VIA;
}
const uint64_t manu = SIMDID_PACK_MANU(manu_code);
uint64_t main_lvl = SIMDID_PACK_MAIN(SIMDID_MAIN_LVL_NONE);
uint64_t avx_bits = 0;
uint64_t avx512_bits = 0;
uint64_t avx10_bits = 0;
uint64_t uarch_bits = 0;
int has_hybrid = 0;
if (has_sse42) {
main_lvl = SIMDID_PACK_MAIN(SIMDID_MAIN_LVL_SSE);
}
if (has_avx && has_osxsave && os_has_xsave_state(XCR0_AVX_STATE)) {
avx_bits |= SIMDID_AVX;
if (is_amd_glue_ymm(vendor, cpuid1_eax) || is_via) {
avx_bits |= SIMDID_AVX_DUAL;
}
if (max_basic_leaf >= 7) {
get_cpuid(7, 0, &eax, &ebx, &ecx, &edx);
const int has_avx2 = (ebx & (1 << 5)) != 0;
const int has_avx512f = (ebx & (1 << 16)) != 0;
const int has_avx512dq = (ebx & (1 << 17)) != 0;
const int has_avx512cd = (ebx & (1 << 28)) != 0;
const int has_avx512bw = (ebx & (1 << 30)) != 0;
const int has_avx512vl = (ebx & (1u << 31)) != 0;
const int has_avx512_vnni = (ecx & (1 << 11)) != 0;
has_hybrid = (edx & (1 << 15)) != 0;
int has_avx10 = 0;
if (eax >= 1) {
get_cpuid(7, 1, &eax, &ebx, &ecx, &edx);
has_avx10 = (edx & (1 << 19)) != 0;
}
if (has_avx2) {
avx_bits |= SIMDID_AVX2;
if (has_fma) {
avx_bits |= SIMDID_AVX_FMA;
}
if (has_f16c) {
avx_bits |= SIMDID_AVX_F16C;
}
main_lvl = SIMDID_PACK_MAIN(SIMDID_MAIN_LVL_AVX2);
}
if (has_avx512f && has_avx512cd && has_avx512dq && has_avx512bw &&
has_avx512vl && os_has_xsave_state(XCR0_AVX512_STATE)) {
avx512_bits |= SIMDID_AVX512_BASE;
if (has_avx512_vnni) {
avx512_bits |= SIMDID_AVX512_VNNI;
}
if (is_intel) {
avx512_bits |= SIMDID_AVX512_IMPL_NATIVE;
} else if (is_amd) {
avx512_bits |= amd_avx512_impl(family, model);
} else {
avx512_bits |= SIMDID_AVX512_IMPL_UNKNOWN;
}
main_lvl = SIMDID_PACK_MAIN(SIMDID_MAIN_LVL_AVX512);
if (has_avx10 && (max_basic_leaf >= 0x24)) {
int a10_eax = 0;
int a10_ebx = 0;
int a10_ecx = 0;
int a10_edx = 0;
int avx10_ver = 0;
get_cpuid(0x24, 0, &a10_eax, &a10_ebx, &a10_ecx, &a10_edx);
avx10_ver = a10_ebx & 0xFF;
if (avx10_ver >= 1) {
unsigned max_avx10_subleaf = (unsigned)a10_eax;
unsigned stored_version = (unsigned)avx10_ver;
if (stored_version > SIMDID_AVX10_VERSION_MASK) {
stored_version = SIMDID_AVX10_VERSION_MASK;
}
avx10_bits = stored_version;
if (max_avx10_subleaf >= 1) {
get_cpuid(0x24, 1, &a10_eax, &a10_ebx,
&a10_ecx, &a10_edx);
if ((a10_ecx & (1 << 2)) != 0) {
avx10_bits |= SIMDID_AVX10_VNNI_INT;
}
}
main_lvl = SIMDID_PACK_MAIN(SIMDID_MAIN_LVL_AVX10);
}
}
}
}
}
if (has_hybrid && (max_basic_leaf >= 0x1A)) {
int h_eax = 0;
int h_ebx = 0;
int h_ecx = 0;
int h_edx = 0;
get_cpuid(0x1A, 0, &h_eax, &h_ebx, &h_ecx, &h_edx);
{
const unsigned core_type = (unsigned)(h_eax >> 24) & 0xFFU;
if (core_type == 0x40U) {
uarch_bits |= SIMDID_UARCH_HYBRID_P;
} else if (core_type == 0x20U) {
uarch_bits |= SIMDID_UARCH_HYBRID_E;
}
}
} else {
uarch_bits |= SIMDID_UARCH_HYBRID_P | SIMDID_UARCH_HYBRID_E;
}
{
const int has_avx_dual = (avx_bits & SIMDID_AVX_DUAL) != 0;
const uint64_t avx512_impl = avx512_bits & SIMDID_AVX512_IMPL_MASK;
const int has_avx512_dual = avx512_impl == SIMDID_AVX512_IMPL_DUAL;
if (has_avx_dual || has_avx512_dual) {
uarch_bits |= SIMDID_UARCH_SIMD_DUAL;
}
}
{
uint64_t final_mask = root;
final_mask |= manu;
final_mask |= main_lvl;
final_mask |= SIMDID_PACK_AVX512(avx512_bits);
final_mask |= SIMDID_PACK_AVX(avx_bits);
final_mask |= SIMDID_PACK_AVX10(avx10_bits);
final_mask |= SIMDID_PACK_UARCH(uarch_bits);
simdid.mask = final_mask;
}
return simdid;
}
SN_API SIMDID SN_DECL sn_get_simd_level_global()
{
if (global_simdid.mask) {
return global_simdid;
}
global_simdid = sn_get_simd_level_local();
return global_simdid;
}
#endif
@@ -0,0 +1,222 @@
#include "sn_target.h"
#if SN_IS_ARCH(SN_ARCH_X86_64)
#include "StgSharpNative.h"
#include "sn_intrinsic.context.x86.h"
/* -----------------------------------------------------------------------
* Field extraction helpers.
*
* SIMDID layout (64 bits, little-endian byte order in make_byte[]):
* [3..0] Root ISA make_byte[0] low nibble
* [7..4] Manufacture make_byte[0] high nibble
* [15..8] MainLevel make_byte[1]
* [31..16] AVX512 Feature make_byte[2..3] (16 bits)
* [39..32] AVX Feature make_byte[4]
* [47..40] AVX10 Feature make_byte[5]
* [55..48] uArchHi make_byte[6]
* [63..56] Reserved make_byte[7]
* ----------------------------------------------------------------------- */
#define FIELD_MAIN(id) ((unsigned char)(id).make_byte[1])
#define FIELD_AVX512(id) \
((uint16_t)((unsigned char)(id).make_byte[2] | ((unsigned char)(id).make_byte[3] << 8)))
#define FIELD_AVX(id) ((unsigned char)(id).make_byte[4])
#define FIELD_AVX10(id) ((unsigned char)(id).make_byte[5])
#define FIELD_UARCH(id) ((unsigned char)(id).make_byte[6])
/* Portable popcount for a 16-bit value. */
static int popcount16(uint16_t v)
{
int c = 0;
while (v) {
v &= v - 1;
c++;
}
return c;
}
/* Map AVX512_IMPL 2-bit field to a strength ordinal for tie-breaking.
* Index = impl value (0=UNKNOWN, 1=DUAL, 2=NATIVE, 3=reserved).
* Strength: DUAL(0) < UNKNOWN(1) < NATIVE(2). */
static const int impl_strength_table[4] = { 1, 0, 2, 1 };
static int impl_strength(uint16_t avx512)
{
unsigned impl = (avx512 >> SIMDID_AVX512_IMPL_SHIFT) & 0x3u;
return impl_strength_table[impl];
}
/* -----------------------------------------------------------------------
* sn_get_unite_simd
*
* Produce the conservative intersection of two SIMDIDs — the strongest
* capability set that is safe to use on BOTH cores:
* - MainLevel : min (only assume what both cores support)
* - AVX512 features : bitwise AND (IMPL field: pick the weaker one)
* - AVX features : bitwise AND
* - AVX10 version : min; discrete features: bitwise AND
* - Root / Manu : keep left (must be same arch in practice)
* - uArchHi : bitwise OR (HYBRID_P/E union; SIMD_DUAL set if either core is dual)
* ----------------------------------------------------------------------- */
SN_API SIMDID SN_DECL sn_get_unite_simd(SIMDID left, SIMDID right)
{
SIMDID out;
out.mask = 0;
/* Root + Manufacture (byte 0): prefer left (same arch assumed) */
unsigned char lb0 = (unsigned char)left.make_byte[0];
unsigned char rb0 = (unsigned char)right.make_byte[0];
out.make_byte[0] = (char)(lb0 ? lb0 : rb0);
/* MainLevel (byte 1): min */
unsigned char lm = FIELD_MAIN(left);
unsigned char rm = FIELD_MAIN(right);
out.make_byte[1] = (char)(lm < rm ? lm : rm);
/* AVX512 features (bytes 2-3): AND all bits, then fix IMPL to weaker */
uint16_t l512 = FIELD_AVX512(left);
uint16_t r512 = FIELD_AVX512(right);
uint16_t merged = l512 & r512;
/* Clear the IMPL field, then set the weaker one */
int ls = impl_strength(l512);
int rs = impl_strength(r512);
int s = (ls <= rs) ? l512 : r512; /* pick the one with weaker strength ordinal */
unsigned weak_impl = (s >> SIMDID_AVX512_IMPL_SHIFT) & 0x3u;
merged &= ~(uint16_t)SIMDID_AVX512_IMPL_MASK;
merged |= (uint16_t)(weak_impl << SIMDID_AVX512_IMPL_SHIFT);
out.make_byte[2] = (char)(merged & 0xFF);
out.make_byte[3] = (char)((merged >> 8) & 0xFF);
/* AVX (byte 4): AND */
out.make_byte[4] = left.make_byte[4] & right.make_byte[4];
/* AVX10 (byte 5): minimum version and common discrete features. */
unsigned char lavx10 = FIELD_AVX10(left);
unsigned char ravx10 = FIELD_AVX10(right);
unsigned char lver = lavx10 & SIMDID_AVX10_VERSION_MASK;
unsigned char rver = ravx10 & SIMDID_AVX10_VERSION_MASK;
unsigned char common = (lavx10 & ravx10) & SIMDID_AVX10_VNNI_INT;
out.make_byte[5] = (char)((lver < rver ? lver : rver) | common);
/* uArchHi (byte 6):
* topology bits [1..0]: AND — conservative intersection (weaker topology wins)
* SIMD_DUAL bit [2]: OR — dual if either core is dual */
unsigned char lu = FIELD_UARCH(left);
unsigned char ru = FIELD_UARCH(right);
unsigned char topo = (lu & 0x3u) & (ru & 0x3u);
unsigned char dual = (lu | ru) & (unsigned char)SIMDID_UARCH_SIMD_DUAL;
out.make_byte[6] = (char)(topo | dual);
/* Reserved tail byte. */
out.make_byte[7] = 0;
return out;
}
/* -----------------------------------------------------------------------
* sn_compare_simd
*
* Compare two SIMDIDs by ISA strength. Returns:
* +1 if left is strictly stronger than right
* -1 if left is strictly weaker than right
* 0 if equal
*
* Comparison priority (most significant first):
*
* 1. MainLevel ([15..8])
* Higher value = stronger. This is the primary discriminator:
* NONE < SSE < AVX2 < AVX512 < AVX10
*
* 2. AVX10 version and discrete features ([47..40])
* Only meaningful when both MainLevel == AVX10.
* Higher versions are stronger; VNNI_INT is a tie-breaker.
*
* 3. AVX512 BASE bit (bit 4 of [31..16])
* Only meaningful when both MainLevel >= AVX512.
* BASE=1 (F+CD+VL+DQ+BW all present) > BASE=0.
*
* 4. AVX512 extension popcount
* VNNI is currently the only tracked AVX512 extension.
* The IMPL and BASE bits are masked out before counting so they don't
* inflate the extension count.
*
* 5. AVX512 IMPL strength (tie-break, performance hint)
* DUAL(weakest) < UNKNOWN < NATIVE(strongest).
* This is not ISA capability but execution-width risk.
*
* 6. uArchHi topology bits [1..0]
* Numeric value: unified(11=3) > P-core(10=2) > E-core(01=1) > unknown(00=0).
* Higher value = stronger / higher-capability core form.
*
* 7. uArchHi SIMD_DUAL (final tie-break)
* Non-dual (SIMD_DUAL=0) > dual (SIMD_DUAL=1).
* ----------------------------------------------------------------------- */
SN_API int SN_DECL sn_compare_simd(SIMDID left, SIMDID right)
{
/* 1. MainLevel */
unsigned lm = FIELD_MAIN(left);
unsigned rm = FIELD_MAIN(right);
if (lm != rm)
return lm > rm ? 1 : -1;
/* 2. AVX10 version and discrete features (only at AVX10). */
if (lm >= SIMDID_MAIN_LVL_AVX10) {
unsigned la = FIELD_AVX10(left);
unsigned ra = FIELD_AVX10(right);
unsigned lv = la & SIMDID_AVX10_VERSION_MASK;
unsigned rv = ra & SIMDID_AVX10_VERSION_MASK;
if (lv != rv)
return lv > rv ? 1 : -1;
int lvnni = (la & SIMDID_AVX10_VNNI_INT) != 0;
int rvnni = (ra & SIMDID_AVX10_VNNI_INT) != 0;
if (lvnni != rvnni)
return lvnni ? 1 : -1;
}
/* 3. AVX512 BASE (only when MainLevel >= AVX512) */
if (lm >= SIMDID_MAIN_LVL_AVX512) {
uint16_t l512 = FIELD_AVX512(left);
uint16_t r512 = FIELD_AVX512(right);
int lb = (l512 & SIMDID_AVX512_BASE) != 0;
int rb = (r512 & SIMDID_AVX512_BASE) != 0;
if (lb != rb)
return lb > rb ? 1 : -1;
// 4. Extension popcount (mask out BASE + IMPL + reserved low nibble)
int drop = ~(SIMDID_AVX512_BASE | SIMDID_AVX512_IMPL_MASK | 0x000Fu);
uint16_t ext_mask = drop & (l512 | r512); /* only consider bits set in either */
int lp = popcount16(l512 & ext_mask);
int rp = popcount16(r512 & ext_mask);
if (lp != rp)
return lp > rp ? 1 : -1;
/* 5. IMPL strength */
int ls = impl_strength(l512);
int rs = impl_strength(r512);
if (ls != rs)
return ls > rs ? 1 : -1;
}
/* 6. uArchHi topology bits [1..0]: unified(3) > P(2) > E(1) > unknown(0) */
unsigned lu = FIELD_UARCH(left);
unsigned ru = FIELD_UARCH(right);
unsigned lt = lu & 0x3u;
unsigned rt = ru & 0x3u;
if (lt != rt)
return lt > rt ? 1 : -1;
/* 7. uArchHi SIMD_DUAL: non-dual (SIMD_DUAL=0) > dual (SIMD_DUAL=1) */
int ld = (lu & SIMDID_UARCH_SIMD_DUAL) != 0;
int rd = (ru & SIMDID_UARCH_SIMD_DUAL) != 0;
if (ld != rd)
return ld ? -1 : 1; /* non-dual is stronger */
return 0;
}
#endif
+332
View File
@@ -0,0 +1,332 @@
#include "sn_target.h"
#if SN_IS_ARCH(SN_ARCH_X86_64)
#include "sn_intrinsic.h"
#include "sn_intrinsic.std.h"
#include "sn_intrinsic.context.x86.h"
#define _FMA_SELECT(i) _MM_SHUFFLE(i, i, i, i)
#define GET_INDEX(col_len, x, y) (col_len * x + y)
// float
SN_SK_PROC_DECL_STD(float, sse, , fma)
{
__sk_param_std(float);
register __m128 c0 = _mm_load_ps((float *)&ans->x[0]);
register __m128 c1 = _mm_load_ps((float *)&ans->x[1]);
register __m128 c2 = _mm_load_ps((float *)&ans->x[2]);
register __m128 c3 = _mm_load_ps((float *)&ans->x[3]);
register __m128 bcol;
const __m128 a0 = _mm_load_ps((float *)&left->x[0]);
const __m128 a1 = _mm_load_ps((float *)&left->x[1]);
const __m128 a2 = _mm_load_ps((float *)&left->x[2]);
const __m128 a3 = _mm_load_ps((float *)&left->x[3]);
/* Column 0 */
bcol = _mm_load_ps((float *)&right->x[0]);
c0 = _mm_add_ps(c0, _mm_mul_ps(a0, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(0))));
c0 = _mm_add_ps(c0, _mm_mul_ps(a1, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(1))));
c0 = _mm_add_ps(c0, _mm_mul_ps(a2, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(2))));
c0 = _mm_add_ps(c0, _mm_mul_ps(a3, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(3))));
/* Column 1 */
bcol = _mm_load_ps((float *)&right->x[1]);
c1 = _mm_add_ps(c1, _mm_mul_ps(a0, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(0))));
c1 = _mm_add_ps(c1, _mm_mul_ps(a1, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(1))));
c1 = _mm_add_ps(c1, _mm_mul_ps(a2, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(2))));
c1 = _mm_add_ps(c1, _mm_mul_ps(a3, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(3))));
/* Column 2 */
bcol = _mm_load_ps((float *)&right->x[2]);
c2 = _mm_add_ps(c2, _mm_mul_ps(a0, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(0))));
c2 = _mm_add_ps(c2, _mm_mul_ps(a1, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(1))));
c2 = _mm_add_ps(c2, _mm_mul_ps(a2, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(2))));
c2 = _mm_add_ps(c2, _mm_mul_ps(a3, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(3))));
/* Column 3 */
bcol = _mm_load_ps((float *)&right->x[3]);
c3 = _mm_add_ps(c3, _mm_mul_ps(a0, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(0))));
c3 = _mm_add_ps(c3, _mm_mul_ps(a1, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(1))));
c3 = _mm_add_ps(c3, _mm_mul_ps(a2, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(2))));
c3 = _mm_add_ps(c3, _mm_mul_ps(a3, _mm_shuffle_ps(bcol, bcol, _FMA_SELECT(3))));
_mm_store_ps((float *)&ans->x[0], c0);
_mm_store_ps((float *)&ans->x[1], c1);
_mm_store_ps((float *)&ans->x[2], c2);
_mm_store_ps((float *)&ans->x[3], c3);
}
SN_SK_PROC_DECL_STD(float, avx, , fma)
{
__sk_param_std(float);
register __m256 c01 = _mm256_load_ps((float *)&ans->y[0]);
register __m256 c23 = _mm256_load_ps((float *)&ans->y[1]);
register __m256 b01 = _mm256_load_ps((float *)&right->y[0]);
register __m256 b23 = _mm256_load_ps((float *)&right->y[1]);
register __m256 a0 = _mm256_broadcast_ps(&left->x[0]);
c01 = _mm256_add_ps(_mm256_mul_ps(a0, b01), c01);
c23 = _mm256_add_ps(_mm256_mul_ps(a0, b23), c23);
register __m256 a1 = _mm256_broadcast_ps(&left->x[1]);
c01 = _mm256_add_ps(_mm256_mul_ps(a1, b01), c01);
c23 = _mm256_add_ps(_mm256_mul_ps(a1, b23), c23);
register __m256 a2 = _mm256_broadcast_ps(&left->x[2]);
c01 = _mm256_add_ps(_mm256_mul_ps(a2, b01), c01);
c23 = _mm256_add_ps(_mm256_mul_ps(a2, b23), c23);
register __m256 a3 = _mm256_broadcast_ps(&left->x[3]);
c01 = _mm256_add_ps(_mm256_mul_ps(a3, b01), c01);
c23 = _mm256_add_ps(_mm256_mul_ps(a3, b23), c23);
_mm256_store_ps((float *)&ans->y[0], c01);
_mm256_store_ps((float *)&ans->y[1], c23);
}
SN_SK_PROC_DECL_STD(float, avx, _fma, fma)
{
__sk_param_std(float);
register __m256 c01 = _mm256_load_ps((float *)&ans->y[0]);
register __m256 c23 = _mm256_load_ps((float *)&ans->y[1]);
register __m256 b01 = _mm256_load_ps((float *)&right->y[0]);
register __m256 b23 = _mm256_load_ps((float *)&right->y[1]);
register __m256 a0 = _mm256_broadcast_ps(&left->x[0]);
c01 = _mm256_fmadd_ps(a0, b01, c01);
c23 = _mm256_fmadd_ps(a0, b23, c23);
register __m256 a1 = _mm256_broadcast_ps(&left->x[1]);
c01 = _mm256_fmadd_ps(a1, b01, c01);
c23 = _mm256_fmadd_ps(a1, b23, c23);
register __m256 a2 = _mm256_broadcast_ps(&left->x[2]);
c01 = _mm256_fmadd_ps(a2, b01, c01);
c23 = _mm256_fmadd_ps(a2, b23, c23);
register __m256 a3 = _mm256_broadcast_ps(&left->x[3]);
c01 = _mm256_fmadd_ps(a3, b01, c01);
c23 = _mm256_fmadd_ps(a3, b23, c23);
_mm256_store_ps((float *)&ans->y[0], c01);
_mm256_store_ps((float *)&ans->y[1], c23);
}
SN_SK_PROC_DECL_STD(float, 512, , fma)
{
__sk_param_std(float);
register __m512 c = _mm512_load_ps((float *)ans);
// full right load:[kb_col0 | kb_col1 | kb_col2 | kb_col3]
register __m512 b = _mm512_load_ps((float *)right);
// k=0: broadcast ka_col0 到4个lane,permute_ps 取 kb(0,j)
register __m512 ak;
ak = _mm512_broadcast_f32x4(left->x[0]);
c = _mm512_fmadd_ps(ak, _mm512_permute_ps(b, _FMA_SELECT(0)), c);
// k=1
ak = _mm512_broadcast_f32x4(left->x[1]);
c = _mm512_fmadd_ps(ak, _mm512_permute_ps(b, _FMA_SELECT(1)), c);
// k=2
ak = _mm512_broadcast_f32x4(left->x[2]);
c = _mm512_fmadd_ps(ak, _mm512_permute_ps(b, _FMA_SELECT(2)), c);
// k=3
ak = _mm512_broadcast_f32x4(left->x[3]);
c = _mm512_fmadd_ps(ak, _mm512_permute_ps(b, _FMA_SELECT(3)), c);
_mm512_store_ps((float *)ans, c);
}
// double
// clang-format off
#define DOUBLE_KERNEL_FMA_CYCLE(i) \
do { \
/* k = 0 */ \
register __m128d b_lo = _mm_load_pd(right_values + (2 * i + 0)); \
register __m128d b_hi = _mm_load_pd(right_values + (2 * i + 1)); \
register __m128d a_lo = _mm_load_pd(left_values + (0 * 2 + 0)); \
register __m128d a_hi = _mm_load_pd(left_values + (0 * 2 + 1)); \
register __m128d b = _mm_shuffle_pd(b_lo, b_lo, 0x0); \
c##i##_lo = _mm_add_pd(c##i##_lo, _mm_mul_pd(a_lo, b)); \
c##i##_hi = _mm_add_pd(c##i##_hi, _mm_mul_pd(a_hi, b)); \
\
/* group = 1 */ \
a_lo = _mm_load_pd(left_values + (1 * 2 + 0)); \
a_hi = _mm_load_pd(left_values + (1 * 2 + 1)); \
b = _mm_shuffle_pd(b_lo, b_lo, 0x3); \
c##i##_lo = _mm_add_pd(c##i##_lo, _mm_mul_pd(a_lo, b)); \
c##i##_hi = _mm_add_pd(c##i##_hi, _mm_mul_pd(a_hi, b)); \
\
/* groupp = 2 */ \
a_lo = _mm_load_pd(left_values + (2 * 2 + 0)); \
a_hi = _mm_load_pd(left_values + (2 * 2 + 1)); \
b = _mm_shuffle_pd(b_hi, b_hi, 0x0); \
c##i##_lo = _mm_add_pd(c##i##_lo, _mm_mul_pd(a_lo, b)); \
c##i##_hi = _mm_add_pd(c##i##_hi, _mm_mul_pd(a_hi, b)); \
\
/* froup = 3 */ \
a_lo = _mm_load_pd(left_values + (3 * 2 + 0)); \
a_hi = _mm_load_pd(left_values + (3 * 2 + 1)); \
b = _mm_shuffle_pd(b_hi, b_hi, 0x3); \
c##i##_lo = _mm_add_pd(c##i##_lo, _mm_mul_pd(a_lo, b)); \
c##i##_hi = _mm_add_pd(c##i##_hi, _mm_mul_pd(a_hi, b)); \
} while (0)
// clang-format on
SN_SK_PROC_DECL_STD(double, sse, , fma)
{
__sk_param_std(double);
const double *left_values = &left->m[0][0];
const double *right_values = &right->m[0][0];
register __m128d c0_lo = ans->x[0];
register __m128d c0_hi = ans->x[1];
register __m128d c1_lo = ans->x[2];
register __m128d c1_hi = ans->x[3];
register __m128d c2_lo = ans->x[4];
register __m128d c2_hi = ans->x[5];
register __m128d c3_lo = ans->x[6];
register __m128d c3_hi = ans->x[7];
DOUBLE_KERNEL_FMA_CYCLE(0);
DOUBLE_KERNEL_FMA_CYCLE(1);
DOUBLE_KERNEL_FMA_CYCLE(2);
DOUBLE_KERNEL_FMA_CYCLE(3);
ans->x[0] = c0_lo;
ans->x[1] = c0_hi;
ans->x[2] = c1_lo;
ans->x[3] = c1_hi;
ans->x[4] = c2_lo;
ans->x[5] = c2_hi;
ans->x[6] = c3_lo;
ans->x[7] = c3_hi;
}
// clang-format off
SN_SK_PROC_DECL_STD(double, avx, , fma)
{
__sk_param_std(double);
register __m256d c0 = _mm256_load_pd((double *)&ans->y[0]);
register __m256d c1 = _mm256_load_pd((double *)&ans->y[1]);
register __m256d c2 = _mm256_load_pd((double *)&ans->y[2]);
register __m256d c3 = _mm256_load_pd((double *)&ans->y[3]);
register __m256d bcol;
const __m256d a0 = _mm256_load_pd((double *)&left->y[0]);
const __m256d a1 = _mm256_load_pd((double *)&left->y[1]);
const __m256d a2 = _mm256_load_pd((double *)&left->y[2]);
const __m256d a3 = _mm256_load_pd((double *)&left->y[3]);
/* Column 0 */
bcol = _mm256_load_pd((double *)&right->y[0]);
c0 = _mm256_fmadd_pd(a0, _mm256_permute4x64_pd(bcol, _FMA_SELECT(0)), c0);
c0 = _mm256_fmadd_pd(a1, _mm256_permute4x64_pd(bcol, _FMA_SELECT(1)), c0);
c0 = _mm256_fmadd_pd(a2, _mm256_permute4x64_pd(bcol, _FMA_SELECT(2)), c0);
c0 = _mm256_fmadd_pd(a3, _mm256_permute4x64_pd(bcol, _FMA_SELECT(3)), c0);
/* Column 1 */
bcol = _mm256_load_pd((double *)&right->y[1]);
c1 = _mm256_fmadd_pd(a0, _mm256_permute4x64_pd(bcol, _FMA_SELECT(0)), c1);
c1 = _mm256_fmadd_pd(a1, _mm256_permute4x64_pd(bcol, _FMA_SELECT(1)), c1);
c1 = _mm256_fmadd_pd(a2, _mm256_permute4x64_pd(bcol, _FMA_SELECT(2)), c1);
c1 = _mm256_fmadd_pd(a3, _mm256_permute4x64_pd(bcol, _FMA_SELECT(3)), c1);
/* Column 2 */
bcol = _mm256_load_pd((double *)&right->y[2]);
c2 = _mm256_fmadd_pd(a0, _mm256_permute4x64_pd(bcol, _FMA_SELECT(0)), c2);
c2 = _mm256_fmadd_pd(a1, _mm256_permute4x64_pd(bcol, _FMA_SELECT(1)), c2);
c2 = _mm256_fmadd_pd(a2, _mm256_permute4x64_pd(bcol, _FMA_SELECT(2)), c2);
c2 = _mm256_fmadd_pd(a3, _mm256_permute4x64_pd(bcol, _FMA_SELECT(3)), c2);
/* Column 3 */
bcol = _mm256_load_pd((double *)&right->y[3]);
c3 = _mm256_fmadd_pd(a0, _mm256_permute4x64_pd(bcol, _FMA_SELECT(0)), c3);
c3 = _mm256_fmadd_pd(a1, _mm256_permute4x64_pd(bcol, _FMA_SELECT(1)), c3);
c3 = _mm256_fmadd_pd(a2, _mm256_permute4x64_pd(bcol, _FMA_SELECT(2)), c3);
c3 = _mm256_fmadd_pd(a3, _mm256_permute4x64_pd(bcol, _FMA_SELECT(3)), c3);
_mm256_store_pd((double *)&ans->y[0], c0);
_mm256_store_pd((double *)&ans->y[1], c1);
_mm256_store_pd((double *)&ans->y[2], c2);
_mm256_store_pd((double *)&ans->y[3], c3);
}
// clang-format on
SN_SK_PROC_DECL_STD(double, avx, _fma, fma)
{
__sk_param_std(double);
register __m256d c0 = _mm256_load_pd((double *)&ans->y[0]);
register __m256d c1 = _mm256_load_pd((double *)&ans->y[1]);
register __m256d c2 = _mm256_load_pd((double *)&ans->y[2]);
register __m256d c3 = _mm256_load_pd((double *)&ans->y[3]);
register __m256d bcol;
const __m256d a0 = _mm256_load_pd((double *)&left->y[0]);
const __m256d a1 = _mm256_load_pd((double *)&left->y[1]);
const __m256d a2 = _mm256_load_pd((double *)&left->y[2]);
const __m256d a3 = _mm256_load_pd((double *)&left->y[3]);
/* Column 0 */
bcol = _mm256_load_pd((double *)&right->y[0]);
c0 = _mm256_fmadd_pd(a0, _mm256_permute4x64_pd(bcol, _FMA_SELECT(0)), c0);
c0 = _mm256_fmadd_pd(a1, _mm256_permute4x64_pd(bcol, _FMA_SELECT(1)), c0);
c0 = _mm256_fmadd_pd(a2, _mm256_permute4x64_pd(bcol, _FMA_SELECT(2)), c0);
c0 = _mm256_fmadd_pd(a3, _mm256_permute4x64_pd(bcol, _FMA_SELECT(3)), c0);
/* Column 1 */
bcol = _mm256_load_pd((double *)&right->y[1]);
c1 = _mm256_fmadd_pd(a0, _mm256_permute4x64_pd(bcol, _FMA_SELECT(0)), c1);
c1 = _mm256_fmadd_pd(a1, _mm256_permute4x64_pd(bcol, _FMA_SELECT(1)), c1);
c1 = _mm256_fmadd_pd(a2, _mm256_permute4x64_pd(bcol, _FMA_SELECT(2)), c1);
c1 = _mm256_fmadd_pd(a3, _mm256_permute4x64_pd(bcol, _FMA_SELECT(3)), c1);
/* Column 2 */
bcol = _mm256_load_pd((double *)&right->y[2]);
c2 = _mm256_fmadd_pd(a0, _mm256_permute4x64_pd(bcol, _FMA_SELECT(0)), c2);
c2 = _mm256_fmadd_pd(a1, _mm256_permute4x64_pd(bcol, _FMA_SELECT(1)), c2);
c2 = _mm256_fmadd_pd(a2, _mm256_permute4x64_pd(bcol, _FMA_SELECT(2)), c2);
c2 = _mm256_fmadd_pd(a3, _mm256_permute4x64_pd(bcol, _FMA_SELECT(3)), c2);
/* Column 3 */
bcol = _mm256_load_pd((double *)&right->y[3]);
c3 = _mm256_fmadd_pd(a0, _mm256_permute4x64_pd(bcol, _FMA_SELECT(0)), c3);
c3 = _mm256_fmadd_pd(a1, _mm256_permute4x64_pd(bcol, _FMA_SELECT(1)), c3);
c3 = _mm256_fmadd_pd(a2, _mm256_permute4x64_pd(bcol, _FMA_SELECT(2)), c3);
c3 = _mm256_fmadd_pd(a3, _mm256_permute4x64_pd(bcol, _FMA_SELECT(3)), c3);
_mm256_store_pd((double *)&ans->y[0], c0);
_mm256_store_pd((double *)&ans->y[1], c1);
_mm256_store_pd((double *)&ans->y[2], c2);
_mm256_store_pd((double *)&ans->y[3], c3);
}
SN_SK_PROC_DECL_STD(double, 512, , fma)
{
__sk_param_std(double);
register __m512d c01 = _mm512_load_pd((double *)&ans->z[0]);
register __m512d c23 = _mm512_load_pd((double *)&ans->z[1]);
register __m512d a01 = _mm512_load_pd((double *)&left->z[0]);
register __m512d a23 = _mm512_load_pd((double *)&left->z[1]);
register __m512d b0 = _mm512_broadcast_f64x4(right->y[0]);
c01 = _mm512_fmadd_pd(a01, b0, c01);
c23 = _mm512_fmadd_pd(a23, b0, c23);
register __m512d b1 = _mm512_broadcast_f64x4(right->y[1]);
c01 = _mm512_fmadd_pd(a01, b1, c01);
c23 = _mm512_fmadd_pd(a23, b1, c23);
register __m512d b2 = _mm512_broadcast_f64x4(right->y[2]);
c01 = _mm512_fmadd_pd(a01, b2, c01);
c23 = _mm512_fmadd_pd(a23, b2, c23);
register __m512d b3 = _mm512_broadcast_f64x4(right->y[3]);
c01 = _mm512_fmadd_pd(a01, b3, c01);
c23 = _mm512_fmadd_pd(a23, b3, c23);
_mm512_store_pd((double *)&ans->z[0], c01);
_mm512_store_pd((double *)&ans->z[1], c23);
}
#endif
+327
View File
@@ -0,0 +1,327 @@
#define _GNU_SOURCE
#include "sn_target.h"
#if SN_IS_OS(SN_OS_LINUX)
#include <errno.h>
#include <inttypes.h>
#include <limits.h>
#include <linux/mempolicy.h>
#include <pthread.h>
#include <sched.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/mman.h>
#include <sys/syscall.h>
#include <unistd.h>
typedef int (*sn_linux_id_visitor)(int value, void *context);
typedef struct sn_linux_counter_context {
int count;
} sn_linux_counter_context;
typedef struct sn_linux_max_context {
int max_value;
} sn_linux_max_context;
typedef struct sn_linux_fill_cpu_context {
uint64_t *internal;
uint32_t *global;
int index;
} sn_linux_fill_cpu_context;
typedef struct sn_linux_fill_node_count_context {
int *count_of_each_numa;
} sn_linux_fill_node_count_context;
static int sn_linux_read_first_line(char const *path, char *buffer, size_t buffer_size)
{
FILE *file = fopen(path, "r");
if (file == NULL)
return 0;
char *line = fgets(buffer, (int)buffer_size, file);
fclose(file);
return line != NULL;
}
static int sn_linux_parse_nonnegative_int(char const **cursor, int *value)
{
char *end = NULL;
uintmax_t parsed_value = 0;
if (**cursor == '-')
return 0;
errno = 0;
parsed_value = strtoumax(*cursor, &end, 10);
if (end == *cursor || errno == ERANGE || parsed_value > (uintmax_t)INT_MAX)
return 0;
*cursor = end;
*value = (int)parsed_value;
return 1;
}
static int sn_linux_visit_id_list(char const *text, sn_linux_id_visitor visitor, void *context)
{
char const *cursor = text;
while (*cursor != '\0') {
while (*cursor == ' ' || *cursor == '\t' || *cursor == '\n' || *cursor == ',')
cursor++;
if (*cursor == '\0')
break;
int begin = 0;
if (!sn_linux_parse_nonnegative_int(&cursor, &begin))
return 0;
int finish = begin;
if (*cursor == '-') {
cursor++;
if (!sn_linux_parse_nonnegative_int(&cursor, &finish))
return 0;
}
if (finish < begin)
return 0;
for (int value = begin;; value++) {
if (!visitor(value, context))
return 0;
if (value == finish)
break;
}
}
return 1;
}
static int sn_linux_count_id(int value, void *context)
{
(void)value;
((sn_linux_counter_context *)context)->count++;
return 1;
}
static int sn_linux_track_max_id(int value, void *context)
{
sn_linux_max_context *max_context = (sn_linux_max_context *)context;
if (value > max_context->max_value)
max_context->max_value = value;
return 1;
}
static int sn_linux_fill_cpu_id(int value, void *context)
{
sn_linux_fill_cpu_context *fill_context = (sn_linux_fill_cpu_context *)context;
if (fill_context->internal != NULL)
fill_context->internal[fill_context->index] = (uint64_t)fill_context->index;
if (fill_context->global != NULL)
fill_context->global[fill_context->index] = (uint32_t)value;
fill_context->index++;
return 1;
}
static int sn_linux_count_cpu_list(char const *path)
{
char buffer[256];
if (!sn_linux_read_first_line(path, buffer, sizeof(buffer)))
return 0;
sn_linux_counter_context counter = { 0 };
if (!sn_linux_visit_id_list(buffer, sn_linux_count_id, &counter))
return 0;
return counter.count;
}
static int sn_linux_fill_node_cpu_count(int value, void *context)
{
char path[128];
snprintf(path, sizeof(path), "/sys/devices/system/node/node%d/cpulist", value);
((sn_linux_fill_node_count_context *)context)->count_of_each_numa[value] =
sn_linux_count_cpu_list(path);
return 1;
}
static int sn_linux_node_exists(int numa_id)
{
char path[128];
snprintf(path, sizeof(path), "/sys/devices/system/node/node%d", numa_id);
return access(path, F_OK) == 0;
}
static size_t sn_linux_round_up_to_page_size(size_t size)
{
int64_t page_size_value = (int64_t)sysconf(_SC_PAGESIZE);
size_t effective_page_size = (page_size_value > 0) ? (size_t)page_size_value : 4096u;
size_t rounded = size + effective_page_size - 1;
if (rounded < size)
return 0;
return rounded - (rounded % effective_page_size);
}
static int sn_linux_bind_memory_to_node(void *address, size_t size, int numa_id, uint32_t flags)
{
#if defined(SYS_mbind)
size_t bits_per_word = sizeof(unsigned long) * 8u;
size_t word_count = ((size_t)numa_id / bits_per_word) + 1u;
unsigned long *node_mask = (unsigned long *)calloc(word_count, sizeof(unsigned long));
if (node_mask == NULL)
return 0;
node_mask[(size_t)numa_id / bits_per_word] = 1ul << ((size_t)numa_id % bits_per_word);
int syscall_result = (int)syscall(SYS_mbind, address, size, MPOL_BIND | MPOL_F_STATIC_NODES,
node_mask, (unsigned long)numa_id + 1ul, flags);
free(node_mask);
return syscall_result == 0;
#else
(void)address;
(void)size;
(void)numa_id;
(void)flags;
return 0;
#endif
}
static int sn_linux_populate_writable_mapping(void *address, size_t size)
{
#if defined(MADV_POPULATE_WRITE)
if (madvise(address, size, MADV_POPULATE_WRITE) == 0)
return 1;
return errno == EINVAL;
#else
(void)address;
(void)size;
return 1;
#endif
}
/* Pin the calling thread to a single logical processor.
* core_id >= 0 : bind to that logical processor (global index, across groups).
* core_id < 0 : clear affinity constraint within the current group. */
SN_API void SN_DECL sn_set_thread_affinity(int core_id, int numa_id)
{
int cpu_count = (int)sysconf(_SC_NPROCESSORS_ONLN);
if (cpu_count <= 0)
return;
cpu_set_t *set = CPU_ALLOC(cpu_count);
size_t size = CPU_ALLOC_SIZE(cpu_count);
if (set == NULL)
return;
CPU_ZERO_S(size, set);
if (core_id < 0) {
for (int i = 0; i < cpu_count; i++)
CPU_SET_S((int)i, size, set);
} else {
CPU_SET_S(core_id, size, set);
}
pthread_setaffinity_np(pthread_self(), size, set);
CPU_FREE(set);
}
SN_API int SN_DECL sn_get_numa_overall(int *out_numa_count, int *count_of_each_numa)
{
char buffer[256];
if (!sn_linux_read_first_line("/sys/devices/system/node/online", buffer, sizeof(buffer)))
return 0;
sn_linux_max_context max_context = { -1 };
if (!sn_linux_visit_id_list(buffer, sn_linux_track_max_id, &max_context) ||
max_context.max_value < 0)
return 0;
if (out_numa_count != NULL)
*out_numa_count = max_context.max_value + 1;
if (count_of_each_numa != NULL) {
for (int index = 0; index <= max_context.max_value; index++)
count_of_each_numa[index] = 0;
sn_linux_fill_node_count_context fill_context = { count_of_each_numa };
if (!sn_linux_visit_id_list(buffer, sn_linux_fill_node_cpu_count, &fill_context))
return 0;
}
return 1;
}
SN_API int SN_DECL sn_get_numa_each(int numa_id, uint64_t *internal, uint32_t *global)
{
if (numa_id < 0)
return 0;
char path[128];
char buffer[256];
snprintf(path, sizeof(path), "/sys/devices/system/node/node%d/cpulist", numa_id);
if (!sn_linux_read_first_line(path, buffer, sizeof(buffer)))
return 0;
sn_linux_fill_cpu_context fill_context = { internal, global, 0 };
return sn_linux_visit_id_list(buffer, sn_linux_fill_cpu_id, &fill_context);
}
SN_API void *SN_DECL sn_numa_alloc(size_t size, int numa_id)
{
size_t allocation_size = sn_linux_round_up_to_page_size(size);
if (allocation_size == 0)
return NULL;
int mmap_flags = MAP_PRIVATE | MAP_ANONYMOUS;
#if defined(MAP_POPULATE)
if (numa_id >= 0)
mmap_flags |= MAP_POPULATE;
#endif
void *memory = mmap(NULL, allocation_size, PROT_READ | PROT_WRITE, mmap_flags, -1, 0);
if (memory == MAP_FAILED)
return NULL;
if (numa_id < 0)
return memory;
uint32_t bind_flags = 0u;
#if defined(MAP_POPULATE)
bind_flags = MPOL_MF_MOVE | MPOL_MF_STRICT;
#endif
if (!sn_linux_node_exists(numa_id) ||
!sn_linux_bind_memory_to_node(memory, allocation_size, numa_id, bind_flags) ||
!sn_linux_populate_writable_mapping(memory, allocation_size)) {
munmap(memory, allocation_size);
return NULL;
}
return memory;
}
SN_API void SN_DECL sn_numa_free(void *ptr, size_t size, int numa_id)
{
(void)numa_id;
if (ptr == NULL || size == 0)
return;
size_t allocation_size = sn_linux_round_up_to_page_size(size);
if (allocation_size != 0)
munmap(ptr, allocation_size);
}
#endif
@@ -0,0 +1,95 @@
#include "sn_target.h"
#include "StgSharpNative.h"
#if SN_IS_OS(SN_OS_WINDOWS)
/* Pin the calling thread to a single logical processor.
* core_id >= 0 : bind to that logical processor (global index, across groups).
* core_id < 0 : clear affinity constraint within the current group. */
SN_API void SN_DECL sn_set_thread_affinity(int core_id, int numa_id)
{
if (core_id < 0 || numa_id < 0) {
/* Query which group the thread currently belongs to */
GROUP_AFFINITY current;
GetThreadGroupAffinity(GetCurrentThread(), &current);
/* Build a full mask for that group's actual core count */
DWORD count = GetActiveProcessorCount(current.Group);
GROUP_AFFINITY ga;
memset(&ga, 0, sizeof(ga));
ga.Group = current.Group;
ga.Mask = (count >= 64) ? ~(KAFFINITY)0 : ((KAFFINITY)1 << count) - 1;
SetThreadGroupAffinity(GetCurrentThread(), &ga, NULL);
return;
}
int index = core_id % 64;
DWORD count = GetActiveProcessorCount(numa_id);
if ((DWORD)index >= count)
return;
GROUP_AFFINITY ga;
memset(&ga, 0, sizeof(ga));
ga.Group = (WORD)numa_id;
ga.Mask = (KAFFINITY)1 << index;
SetThreadGroupAffinity(GetCurrentThread(), &ga, NULL);
}
SN_API int SN_DECL sn_get_numa_overall(int *out_numa_count, int *count_of_each_numa)
{
ULONG highest;
if (!GetNumaHighestNodeNumber(&highest))
return 0;
*out_numa_count = (int)highest + 1;
for (UCHAR n = 0; n <= (UCHAR)highest; n++) {
ULONGLONG mask = 0;
GetNumaNodeProcessorMask(n, &mask);
count_of_each_numa[n] = (int)__popcnt64(mask);
}
return 1;
}
SN_API int SN_DECL sn_get_numa_each(int numa_id, uint64_t *internal, uint32_t *global)
{
ULONGLONG mask = 0;
if (!GetNumaNodeProcessorMask((UCHAR)numa_id, &mask))
return 0;
int intra = 0;
while (mask) {
DWORD idx;
_BitScanForward64(&idx, mask);
if (internal != NULL)
internal[intra] = (uint64_t)intra;
global[intra] = (uint32_t)idx;
intra++;
mask &= mask - 1;
}
return 1;
}
SN_API void *SN_DECL sn_numa_alloc(size_t size, int numa_id)
{
if (size == 0)
return NULL;
if (numa_id < 0)
return VirtualAlloc(NULL, size, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE);
ULONG highest = 0;
if (!GetNumaHighestNodeNumber(&highest) || (ULONG)numa_id > highest)
return NULL;
return VirtualAllocExNuma(GetCurrentProcess(), NULL, size, MEM_RESERVE | MEM_COMMIT,
PAGE_READWRITE, (DWORD)numa_id);
}
SN_API void SN_DECL sn_numa_free(void *ptr, size_t size, int numa_id)
{
(void)size;
(void)numa_id;
if (ptr != NULL)
VirtualFree(ptr, 0, MEM_RELEASE);
}
#endif
+32
View File
@@ -0,0 +1,32 @@
#include "sn_intrinsic.h"
#include "sn_internal.h"
INTERNAL void SN_DECL f32_normalize(VEC_SEGMENT(float) *restrict source, VEC_SEGMENT(float) *restrict target)
{
register __m128 s = source->f32_x[0];
register __m128 a_length_sq = _mm_mul_ps(s, s);
a_length_sq = _mm_add_ps(a_length_sq,
_mm_shuffle_ps(a_length_sq, a_length_sq, _MM_SHUFFLE(2, 3, 0, 1)));
a_length_sq = _mm_add_ss(a_length_sq,
_mm_shuffle_ps(a_length_sq, a_length_sq, _MM_SHUFFLE(0, 0, 3, 2)));
__m128 a_inv_length = _mm_rsqrt_ps(a_length_sq);
s = _mm_mul_ps(s, a_inv_length);
__m128 dot_product = _mm_mul_ps(target->f32_x[0], s);
dot_product = _mm_add_ps(dot_product,
_mm_shuffle_ps(dot_product, dot_product, _MM_SHUFFLE(2, 3, 0, 1)));
dot_product = _mm_add_ss(dot_product,
_mm_shuffle_ps(dot_product, dot_product, _MM_SHUFFLE(0, 0, 3, 2)));
__m128 b_orthogonal = _mm_sub_ps(target->f32_x[0], _mm_mul_ps(s, dot_product));
__m128 b_length_sq = _mm_mul_ps(b_orthogonal, b_orthogonal);
b_length_sq = _mm_add_ps(b_length_sq,
_mm_shuffle_ps(b_length_sq, b_length_sq, _MM_SHUFFLE(2, 3, 0, 1)));
b_length_sq = _mm_add_ss(b_length_sq,
_mm_shuffle_ps(b_length_sq, b_length_sq, _MM_SHUFFLE(0, 0, 3, 2)));
__m128 b_inv_length = _mm_rsqrt_ps(b_length_sq);
target->f32_x[0] = _mm_mul_ps(b_orthogonal, b_inv_length);
}
+67
View File
@@ -0,0 +1,67 @@
#include "sn_intrinsic.h"
#include "sn_internal.h"
static const int avxdot_indices[] = { 0, 4, 1, 5, 2, 3, 6, 7 };
INTERNAL void FORCEINLINE SN_DECL dot_41_sse(MAT_KERNEL(float) const *transpose,
__m128 const *vector, __m128 *ans)
{
*ans = _mm_dp_ps(transpose->x[0], *vector, 0B11110001);
*ans = _mm_insert_ps(*ans, _mm_dp_ps(transpose->x[1], *vector, 0B11110001),
SSE_INSERT_SIMPLE(1, 1));
*ans = _mm_insert_ps(*ans, _mm_dp_ps(transpose->x[2], *vector, 0B11110001),
SSE_INSERT_SIMPLE(1, 3));
*ans = _mm_insert_ps(*ans, _mm_dp_ps(transpose->x[3], *vector, 0B11110001),
SSE_INSERT_SIMPLE(1, 4));
}
INTERNAL void SN_DECL dot_41_avx(MAT_KERNEL(float) const *transpose, __m128 const *vector,
__m128 *ans)
{
__m256 _double = _mm256_set_m128(vector[0], vector[0]);
__m256 tmp = _mm256_dp_ps(transpose->y[0], _double, 0B11110001);
tmp = _mm256_blend_ps(tmp, _mm256_dp_ps(transpose->y[0], _double, 0B11110010),
0B00100010);
*ans = _mm256_castps256_ps128(
_mm256_permutevar8x32_ps(tmp, _mm256_loadu_epi32(avxdot_indices)));
}
INTERNAL void SN_DECL dot_42_sse(MAT_KERNEL(float) const *transpose, __m128 const *vector,
__m128 *ans)
{
ans[0] = _mm_dp_ps(transpose->x[0], vector[0], 0B11110001);
ans[0] = _mm_insert_ps(*ans, _mm_dp_ps(transpose->x[1], vector[0], 0B11110001),
SSE_INSERT_SIMPLE(1, 2));
ans[0] = _mm_insert_ps(*ans, _mm_dp_ps(transpose->x[2], vector[0], 0B11110001),
SSE_INSERT_SIMPLE(1, 3));
ans[0] = _mm_insert_ps(*ans, _mm_dp_ps(transpose->x[3], vector[0], 0B11110001),
SSE_INSERT_SIMPLE(1, 4));
ans[1] = _mm_dp_ps(transpose->x[0], vector[1], 0B11110001);
ans[1] = _mm_insert_ps(*ans, _mm_dp_ps(transpose->x[1], vector[1], 0B11110001),
SSE_INSERT_SIMPLE(1, 2));
ans[1] = _mm_insert_ps(*ans, _mm_dp_ps(transpose->x[2], vector[1], 0B11110001),
SSE_INSERT_SIMPLE(1, 3));
ans[1] = _mm_insert_ps(*ans, _mm_dp_ps(transpose->x[3], vector[1], 0B11110001),
SSE_INSERT_SIMPLE(1, 4));
}
INTERNAL void SN_DECL dot_42_avx(MAT_KERNEL(float) const *transpose, __m128 const *vector,
__m128 *ans)
{
__m256 temp = transpose->y[0];
__m256 vector_stream = _mm256_set_m128(vector[1], vector[0]);
__m256 result = _mm256_dp_ps(temp, vector_stream, 0B11110001);
temp = _mm256_permute2f128_ps(temp, temp, 0B00000001);
result = _mm256_blend_ps(result, _mm256_dp_ps(temp, vector_stream, 0B11110010), 0B00100010);
temp = transpose->y[1];
result = _mm256_blend_ps(result, _mm256_dp_ps(temp, vector_stream, 0B11110100), 0B01000100);
temp = _mm256_permute2f128_ps(temp, temp, 0B00000001);
result = _mm256_blend_ps(result, _mm256_dp_ps(temp, vector_stream, 0B11111000), 0B10001000);
ans[0] = _mm256_castps256_ps128(result);
ans[1] = _mm256_extractf128_ps(result, 1);
ans[1] = _mm_shuffle_ps(ans[1], ans[1], _MM_SHUFFLE(2, 3, 0, 1));
}
@@ -0,0 +1,90 @@
#include "sn_target.h"
#if SN_IS_ARCH(SN_ARCH_X86_64)
#define SN_MK_FMA_F32_512_CYCLE(i) \
do { \
c0 = _mm512_fmadd_ps(c0, a0, _mm512_permute_ps(b0, _FMA_SELECT(i))); \
c1 = _mm512_fmadd_ps(c1, a1, _mm512_permute_ps(b1, _FMA_SELECT(i))); \
c2 = _mm512_fmadd_ps(c2, a2, _mm512_permute_ps(b2, _FMA_SELECT(i))); \
c3 = _mm512_fmadd_ps(c3, a3, _mm512_permute_ps(b3, _FMA_SELECT(i))); \
} while (0)
#define SN_MK_FMA_F32_AVX2_CYCLE(i) \
do { \
c0 = _mm256_fmadd_ps(c0, a0, _mm256_permute_ps(b0, _FMA_SELECT(i))); \
c1 = _mm256_fmadd_ps(c1, a1, _mm256_permute_ps(b1, _FMA_SELECT(i))); \
c2 = _mm256_fmadd_ps(c2, a2, _mm256_permute_ps(b2, _FMA_SELECT(i))); \
c3 = _mm256_fmadd_ps(c3, a3, _mm256_permute_ps(b3, _FMA_SELECT(i))); \
} while (0)
#define SN_MK_FMA_F64_AVX2_CYCLE(i) \
do { \
c0 = _mm512_fmadd_ps(c0, a0, _mm512_permutex_pd(b0, _FMA_SELECT(i))); \
c1 = _mm512_fmadd_ps(c1, a1, _mm512_permutex_pd(b1, _FMA_SELECT(i))); \
c2 = _mm512_fmadd_ps(c2, a2, _mm512_permutex_pd(b2, _FMA_SELECT(i))); \
c3 = _mm512_fmadd_ps(c3, a3, _mm512_permutex_pd(b3, _FMA_SELECT(i))); \
} while (0)
#define SN_FMA_F32_512_PACK_COLUMN(col, target) \
do { \
register __m256 low = _mm512_castps512_ps256(col); \
register __m256 high = _mm512_extractf32x8_ps(col, 1); \
register __m256 half = _mm256_add_ps(high, low); \
register __m128 high128 = _mm256_extractf32x4_ps(half, 1); \
register __m128 low128 = _mm256_castps256_ps128(half); \
register __m128 half128 = _mm_add_ps(high128, low128); \
_mm_store_ps(target, half128); \
} while (0)
#define SN_FMA_F32_AVX2_PACK_COLUMN(col, target) \
do { \
register __m128 high128 = _mm256_extractf32x4_ps(col, 1); \
register __m128 low128 = _mm256_castps256_ps128(col); \
register __m128 half128 = _mm_add_ps(high128, low128); \
_mm_store_ps(target, half128); \
} while (0)
#define SN_FMA_F64_512_PACK_COLUMN(col, target) \
do { \
register __m128 high128 = _mm256_extractf32x4_ps(col, 1); \
register __m128 low128 = _mm256_castps256_ps128(col); \
register __m128 half128 = _mm_add_ps(high128, low128); \
_mm_store_ps(target, half128); \
} while (0)
// double
#define SN_MK_FMA_F64_SSE_PHASE(i) \
do { \
/* k = 0 */ \
register __m128d b_lo = _mm_load_pd(right_values + (2 * i + 0)); \
register __m128d b_hi = _mm_load_pd(right_values + (2 * i + 1)); \
register __m128d a_lo = _mm_load_pd(left_values + (0 * 2 + 0)); \
register __m128d a_hi = _mm_load_pd(left_values + (0 * 2 + 1)); \
register __m128d b = _mm_shuffle_pd(b_lo, b_lo, 0x0); \
c##i##_lo = _mm_add_pd(c##i##_lo, _mm_mul_pd(a_lo, b)); \
c##i##_hi = _mm_add_pd(c##i##_hi, _mm_mul_pd(a_hi, b)); \
\
/* group = 1 */ \
a_lo = _mm_load_pd(left_values + (1 * 2 + 0)); \
a_hi = _mm_load_pd(left_values + (1 * 2 + 1)); \
b = _mm_shuffle_pd(b_lo, b_lo, 0x3); \
c##i##_lo = _mm_add_pd(c##i##_lo, _mm_mul_pd(a_lo, b)); \
c##i##_hi = _mm_add_pd(c##i##_hi, _mm_mul_pd(a_hi, b)); \
\
/* group = 2 */ \
a_lo = _mm_load_pd(left_values + (2 * 2 + 0)); \
a_hi = _mm_load_pd(left_values + (2 * 2 + 1)); \
b = _mm_shuffle_pd(b_hi, b_hi, 0x0); \
c##i##_lo = _mm_add_pd(c##i##_lo, _mm_mul_pd(a_lo, b)); \
c##i##_hi = _mm_add_pd(c##i##_hi, _mm_mul_pd(a_hi, b)); \
\
/* group = 3 */ \
a_lo = _mm_load_pd(left_values + (3 * 2 + 0)); \
a_hi = _mm_load_pd(left_values + (3 * 2 + 1)); \
b = _mm_shuffle_pd(b_hi, b_hi, 0x3); \
c##i##_lo = _mm_add_pd(c##i##_lo, _mm_mul_pd(a_lo, b)); \
c##i##_hi = _mm_add_pd(c##i##_hi, _mm_mul_pd(a_hi, b)); \
} while (0)
#endif