| /* -*- c++ -*- */ |
| /* |
| * Copyright © 2010-2015 Intel Corporation |
| * |
| * Permission is hereby granted, free of charge, to any person obtaining a |
| * copy of this software and associated documentation files (the "Software"), |
| * to deal in the Software without restriction, including without limitation |
| * the rights to use, copy, modify, merge, publish, distribute, sublicense, |
| * and/or sell copies of the Software, and to permit persons to whom the |
| * Software is furnished to do so, subject to the following conditions: |
| * |
| * The above copyright notice and this permission notice (including the next |
| * paragraph) shall be included in all copies or substantial portions of the |
| * Software. |
| * |
| * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
| * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
| * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL |
| * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER |
| * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING |
| * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS |
| * IN THE SOFTWARE. |
| */ |
| |
| #pragma once |
| |
| #include "brw_ir.h" |
| #include "brw_ir_allocator.h" |
| |
| struct fs_inst : public exec_node { |
| private: |
| fs_inst &operator=(const fs_inst &); |
| |
| void init(enum opcode opcode, uint8_t exec_width, const brw_reg &dst, |
| const brw_reg *src, unsigned sources); |
| |
| public: |
| DECLARE_RALLOC_CXX_OPERATORS(fs_inst) |
| |
| fs_inst(); |
| fs_inst(enum opcode opcode, uint8_t exec_size); |
| fs_inst(enum opcode opcode, uint8_t exec_size, const brw_reg &dst); |
| fs_inst(enum opcode opcode, uint8_t exec_size, const brw_reg &dst, |
| const brw_reg &src0); |
| fs_inst(enum opcode opcode, uint8_t exec_size, const brw_reg &dst, |
| const brw_reg &src0, const brw_reg &src1); |
| fs_inst(enum opcode opcode, uint8_t exec_size, const brw_reg &dst, |
| const brw_reg &src0, const brw_reg &src1, const brw_reg &src2); |
| fs_inst(enum opcode opcode, uint8_t exec_size, const brw_reg &dst, |
| const brw_reg src[], unsigned sources); |
| fs_inst(const fs_inst &that); |
| ~fs_inst(); |
| |
| void resize_sources(uint8_t num_sources); |
| |
| bool is_send_from_grf() const; |
| bool is_payload(unsigned arg) const; |
| bool is_partial_write() const; |
| unsigned components_read(unsigned i) const; |
| unsigned size_read(const struct intel_device_info *devinfo, int arg) const; |
| bool can_do_source_mods(const struct intel_device_info *devinfo) const; |
| bool can_do_cmod() const; |
| bool can_change_types() const; |
| bool has_source_and_destination_hazard() const; |
| |
| bool is_3src(const struct brw_compiler *compiler) const; |
| bool is_math() const; |
| bool is_control_flow_begin() const; |
| bool is_control_flow_end() const; |
| bool is_control_flow() const; |
| bool is_commutative() const; |
| bool is_raw_move() const; |
| bool can_do_saturate() const; |
| bool reads_accumulator_implicitly() const; |
| bool writes_accumulator_implicitly(const struct intel_device_info *devinfo) const; |
| |
| /** |
| * Instructions that use indirect addressing have additional register |
| * regioning restrictions. |
| */ |
| bool uses_indirect_addressing() const; |
| |
| void remove(bblock_t *block, bool defer_later_block_ip_updates = false); |
| void insert_after(bblock_t *block, fs_inst *inst); |
| void insert_before(bblock_t *block, fs_inst *inst); |
| |
| /** |
| * True if the instruction has side effects other than writing to |
| * its destination registers. You are expected not to reorder or |
| * optimize these out unless you know what you are doing. |
| */ |
| bool has_side_effects() const; |
| |
| /** |
| * True if the instruction might be affected by side effects of other |
| * instructions. |
| */ |
| bool is_volatile() const; |
| |
| /** |
| * Return whether \p arg is a control source of a virtual instruction which |
| * shouldn't contribute to the execution type and usual regioning |
| * restriction calculations of arithmetic instructions. |
| */ |
| bool is_control_source(unsigned arg) const; |
| |
| /** |
| * Return the subset of flag registers read by the instruction as a bitset |
| * with byte granularity. |
| */ |
| unsigned flags_read(const intel_device_info *devinfo) const; |
| |
| /** |
| * Return the subset of flag registers updated by the instruction (either |
| * partially or fully) as a bitset with byte granularity. |
| */ |
| unsigned flags_written(const intel_device_info *devinfo) const; |
| |
| /** |
| * Return true if this instruction is a sampler message gathering residency |
| * data. |
| */ |
| bool has_sampler_residency() const; |
| |
| /** |
| * Return true if this instruction is using the address register |
| * implicitly. |
| */ |
| bool uses_address_register_implicitly() const; |
| |
| uint8_t sources; /**< Number of brw_reg sources. */ |
| |
| /** |
| * Execution size of the instruction. This is used by the generator to |
| * generate the correct binary for the given instruction. Current valid |
| * values are 1, 4, 8, 16, 32. |
| */ |
| uint8_t exec_size; |
| |
| /** |
| * Channel group from the hardware execution and predication mask that |
| * should be applied to the instruction. The subset of channel enable |
| * signals (calculated from the EU control flow and predication state) |
| * given by [group, group + exec_size) will be used to mask GRF writes and |
| * any other side effects of the instruction. |
| */ |
| uint8_t group; |
| |
| uint8_t mlen; /**< SEND message length */ |
| uint8_t ex_mlen; /**< SENDS extended message length */ |
| uint8_t sfid; /**< SFID for SEND instructions */ |
| /** The number of hardware registers used for a message header. */ |
| uint8_t header_size; |
| uint8_t target; /**< MRT target. */ |
| uint32_t desc; /**< SEND[S] message descriptor immediate */ |
| uint32_t ex_desc; /**< SEND[S] extended message descriptor immediate */ |
| |
| uint32_t offset; /**< spill/unspill offset or texture offset bitfield */ |
| unsigned size_written; /**< Data written to the destination register in bytes. */ |
| |
| enum opcode opcode; /* BRW_OPCODE_* or FS_OPCODE_* */ |
| enum brw_conditional_mod conditional_mod; /**< BRW_CONDITIONAL_* */ |
| enum brw_predicate predicate; |
| |
| tgl_swsb sched; /**< Scheduling info. */ |
| |
| union { |
| struct { |
| /* Chooses which flag subregister (f0.0 to f3.1) is used for |
| * conditional mod and predication. |
| */ |
| unsigned flag_subreg:3; |
| |
| /** |
| * Systolic depth used by DPAS instruction. |
| */ |
| unsigned sdepth:4; |
| |
| /** |
| * Repeat count used by DPAS instruction. |
| */ |
| unsigned rcount:4; |
| |
| unsigned pad:2; |
| |
| bool predicate_inverse:1; |
| bool writes_accumulator:1; /**< instruction implicitly writes accumulator */ |
| bool force_writemask_all:1; |
| bool no_dd_clear:1; |
| bool no_dd_check:1; |
| bool saturate:1; |
| bool shadow_compare:1; |
| bool check_tdr:1; /**< Only valid for SEND; turns it into a SENDC */ |
| bool send_has_side_effects:1; /**< Only valid for SHADER_OPCODE_SEND */ |
| bool send_is_volatile:1; /**< Only valid for SHADER_OPCODE_SEND */ |
| bool send_ex_bso:1; /**< Only for SHADER_OPCODE_SEND, use extended |
| * bindless surface offset (26bits instead of |
| * 20bits) |
| */ |
| /** |
| * The predication mask applied to this instruction is guaranteed to |
| * be uniform and a superset of the execution mask of the present block. |
| * No currently enabled channel will be disabled by the predicate. |
| */ |
| bool predicate_trivial:1; |
| bool eot:1; |
| bool last_rt:1; |
| bool pi_noperspective:1; /**< Pixel interpolator noperspective flag */ |
| bool keep_payload_trailing_zeros:1; |
| /** |
| * Whether the parameters of the SEND instructions are build with |
| * NoMask (for A32 messages this covers only the surface handle, for |
| * A64 messages this covers the load address). |
| */ |
| bool has_no_mask_send_params:1; |
| }; |
| uint32_t bits; |
| }; |
| |
| brw_reg dst; |
| brw_reg *src; |
| brw_reg builtin_src[4]; |
| |
| #ifndef NDEBUG |
| /** @{ |
| * Annotation for the generated IR. |
| */ |
| const char *annotation; |
| /** @} */ |
| #endif |
| }; |
| |
| /** |
| * Make the execution of \p inst dependent on the evaluation of a possibly |
| * inverted predicate. |
| */ |
| static inline fs_inst * |
| set_predicate_inv(enum brw_predicate pred, bool inverse, |
| fs_inst *inst) |
| { |
| inst->predicate = pred; |
| inst->predicate_inverse = inverse; |
| return inst; |
| } |
| |
| /** |
| * Make the execution of \p inst dependent on the evaluation of a predicate. |
| */ |
| static inline fs_inst * |
| set_predicate(enum brw_predicate pred, fs_inst *inst) |
| { |
| return set_predicate_inv(pred, false, inst); |
| } |
| |
| /** |
| * Write the result of evaluating the condition given by \p mod to a flag |
| * register. |
| */ |
| static inline fs_inst * |
| set_condmod(enum brw_conditional_mod mod, fs_inst *inst) |
| { |
| inst->conditional_mod = mod; |
| return inst; |
| } |
| |
| /** |
| * Clamp the result of \p inst to the saturation range of its destination |
| * datatype. |
| */ |
| static inline fs_inst * |
| set_saturate(bool saturate, fs_inst *inst) |
| { |
| inst->saturate = saturate; |
| return inst; |
| } |
| |
| /** |
| * Return the number of dataflow registers written by the instruction (either |
| * fully or partially) counted from 'floor(reg_offset(inst->dst) / |
| * register_size)'. The somewhat arbitrary register size unit is 4B for the |
| * UNIFORM and IMM files and 32B for all other files. |
| */ |
| inline unsigned |
| regs_written(const fs_inst *inst) |
| { |
| assert(inst->dst.file != UNIFORM && inst->dst.file != IMM); |
| return DIV_ROUND_UP(reg_offset(inst->dst) % REG_SIZE + |
| inst->size_written - |
| MIN2(inst->size_written, reg_padding(inst->dst)), |
| REG_SIZE); |
| } |
| |
| /** |
| * Return the number of dataflow registers read by the instruction (either |
| * fully or partially) counted from 'floor(reg_offset(inst->src[i]) / |
| * register_size)'. The somewhat arbitrary register size unit is 4B for the |
| * UNIFORM files and 32B for all other files. |
| */ |
| inline unsigned |
| regs_read(const struct intel_device_info *devinfo, const fs_inst *inst, unsigned i) |
| { |
| if (inst->src[i].file == IMM) |
| return 1; |
| |
| const unsigned reg_size = inst->src[i].file == UNIFORM ? 4 : REG_SIZE; |
| return DIV_ROUND_UP(reg_offset(inst->src[i]) % reg_size + |
| inst->size_read(devinfo, i) - |
| MIN2(inst->size_read(devinfo, i), reg_padding(inst->src[i])), |
| reg_size); |
| } |
| |
| static inline enum brw_reg_type |
| get_exec_type(const fs_inst *inst) |
| { |
| brw_reg_type exec_type = BRW_TYPE_B; |
| |
| for (int i = 0; i < inst->sources; i++) { |
| if (inst->src[i].file != BAD_FILE && |
| !inst->is_control_source(i)) { |
| const brw_reg_type t = get_exec_type(inst->src[i].type); |
| if (brw_type_size_bytes(t) > brw_type_size_bytes(exec_type)) |
| exec_type = t; |
| else if (brw_type_size_bytes(t) == brw_type_size_bytes(exec_type) && |
| brw_type_is_float(t)) |
| exec_type = t; |
| } |
| } |
| |
| if (exec_type == BRW_TYPE_B) |
| exec_type = inst->dst.type; |
| |
| assert(exec_type != BRW_TYPE_B); |
| |
| /* Promotion of the execution type to 32-bit for conversions from or to |
| * half-float seems to be consistent with the following text from the |
| * Cherryview PRM Vol. 7, "Execution Data Type": |
| * |
| * "When single precision and half precision floats are mixed between |
| * source operands or between source and destination operand [..] single |
| * precision float is the execution datatype." |
| * |
| * and from "Register Region Restrictions": |
| * |
| * "Conversion between Integer and HF (Half Float) must be DWord aligned |
| * and strided by a DWord on the destination." |
| */ |
| if (brw_type_size_bytes(exec_type) == 2 && |
| inst->dst.type != exec_type) { |
| if (exec_type == BRW_TYPE_HF) |
| exec_type = BRW_TYPE_F; |
| else if (inst->dst.type == BRW_TYPE_HF) |
| exec_type = BRW_TYPE_D; |
| } |
| |
| return exec_type; |
| } |
| |
| static inline unsigned |
| get_exec_type_size(const fs_inst *inst) |
| { |
| return brw_type_size_bytes(get_exec_type(inst)); |
| } |
| |
| static inline bool |
| is_send(const fs_inst *inst) |
| { |
| return inst->mlen || inst->is_send_from_grf(); |
| } |
| |
| /** |
| * Return whether the instruction isn't an ALU instruction and cannot be |
| * assumed to complete in-order. |
| */ |
| static inline bool |
| is_unordered(const intel_device_info *devinfo, const fs_inst *inst) |
| { |
| return is_send(inst) || (devinfo->ver < 20 && inst->is_math()) || |
| inst->opcode == BRW_OPCODE_DPAS || |
| (devinfo->has_64bit_float_via_math_pipe && |
| (get_exec_type(inst) == BRW_TYPE_DF || |
| inst->dst.type == BRW_TYPE_DF)); |
| } |
| |
| /** |
| * Return whether the following regioning restriction applies to the specified |
| * instruction. From the Cherryview PRM Vol 7. "Register Region |
| * Restrictions": |
| * |
| * "When source or destination datatype is 64b or operation is integer DWord |
| * multiply, regioning in Align1 must follow these rules: |
| * |
| * 1. Source and Destination horizontal stride must be aligned to the same qword. |
| * 2. Regioning must ensure Src.Vstride = Src.Width * Src.Hstride. |
| * 3. Source and Destination offset must be the same, except the case of |
| * scalar source." |
| */ |
| static inline bool |
| has_dst_aligned_region_restriction(const intel_device_info *devinfo, |
| const fs_inst *inst, |
| brw_reg_type dst_type) |
| { |
| const brw_reg_type exec_type = get_exec_type(inst); |
| /* Even though the hardware spec claims that "integer DWord multiply" |
| * operations are restricted, empirical evidence and the behavior of the |
| * simulator suggest that only 32x32-bit integer multiplication is |
| * restricted. |
| */ |
| const bool is_dword_multiply = !brw_type_is_float(exec_type) && |
| ((inst->opcode == BRW_OPCODE_MUL && |
| MIN2(brw_type_size_bytes(inst->src[0].type), brw_type_size_bytes(inst->src[1].type)) >= 4) || |
| (inst->opcode == BRW_OPCODE_MAD && |
| MIN2(brw_type_size_bytes(inst->src[1].type), brw_type_size_bytes(inst->src[2].type)) >= 4)); |
| |
| if (brw_type_size_bytes(dst_type) > 4 || brw_type_size_bytes(exec_type) > 4 || |
| (brw_type_size_bytes(exec_type) == 4 && is_dword_multiply)) |
| return intel_device_info_is_9lp(devinfo) || devinfo->verx10 >= 125; |
| |
| else if (brw_type_is_float(dst_type)) |
| return devinfo->verx10 >= 125; |
| |
| else |
| return false; |
| } |
| |
| static inline bool |
| has_dst_aligned_region_restriction(const intel_device_info *devinfo, |
| const fs_inst *inst) |
| { |
| return has_dst_aligned_region_restriction(devinfo, inst, inst->dst.type); |
| } |
| |
| /** |
| * Return true if the instruction can be potentially affected by the Xe2+ |
| * regioning restrictions that apply to integer types smaller than a dword. |
| * The restriction isn't quoted here due to its length, see BSpec #56640 for |
| * details. |
| */ |
| static inline bool |
| has_subdword_integer_region_restriction(const intel_device_info *devinfo, |
| const fs_inst *inst, |
| const brw_reg *srcs, unsigned num_srcs) |
| { |
| if (devinfo->ver >= 20 && |
| brw_type_is_int(inst->dst.type) && |
| MAX2(byte_stride(inst->dst), |
| brw_type_size_bytes(inst->dst.type)) < 4) { |
| for (unsigned i = 0; i < num_srcs; i++) { |
| if (brw_type_is_int(srcs[i].type) && |
| ((brw_type_size_bytes(srcs[i].type) < 4 && |
| byte_stride(srcs[i]) >= 4) || |
| (MAX2(byte_stride(inst->dst), |
| brw_type_size_bytes(inst->dst.type)) == 1 && |
| brw_type_size_bytes(srcs[i].type) == 1 && |
| byte_stride(srcs[i]) >= 2))) |
| return true; |
| } |
| } |
| |
| return false; |
| } |
| |
| static inline bool |
| has_subdword_integer_region_restriction(const intel_device_info *devinfo, |
| const fs_inst *inst) |
| { |
| return has_subdword_integer_region_restriction(devinfo, inst, |
| inst->src, inst->sources); |
| } |
| |
| /** |
| * Return whether the LOAD_PAYLOAD instruction is a plain copy of bits from |
| * the specified register file into a VGRF. |
| * |
| * This implies identity register regions without any source-destination |
| * overlap, but otherwise has no implications on the location of sources and |
| * destination in the register file: Gathering any number of portions from |
| * multiple virtual registers in any order is allowed. |
| */ |
| inline bool |
| is_copy_payload(const struct intel_device_info *devinfo, |
| brw_reg_file file, const fs_inst *inst) |
| { |
| if (inst->opcode != SHADER_OPCODE_LOAD_PAYLOAD || |
| inst->is_partial_write() || inst->saturate || |
| inst->dst.file != VGRF) |
| return false; |
| |
| for (unsigned i = 0; i < inst->sources; i++) { |
| if (inst->src[i].file != file || |
| inst->src[i].abs || inst->src[i].negate) |
| return false; |
| |
| if (!inst->src[i].is_contiguous()) |
| return false; |
| |
| if (regions_overlap(inst->dst, inst->size_written, |
| inst->src[i], inst->size_read(devinfo, i))) |
| return false; |
| } |
| |
| return true; |
| } |
| |
| /** |
| * Like is_copy_payload(), but the instruction is required to copy a single |
| * contiguous block of registers from the given register file into the |
| * destination without any reordering. |
| */ |
| inline bool |
| is_identity_payload(const struct intel_device_info *devinfo, |
| brw_reg_file file, const fs_inst *inst) |
| { |
| if (is_copy_payload(devinfo, file, inst)) { |
| brw_reg reg = inst->src[0]; |
| |
| for (unsigned i = 0; i < inst->sources; i++) { |
| reg.type = inst->src[i].type; |
| if (!inst->src[i].equals(reg)) |
| return false; |
| |
| reg = byte_offset(reg, inst->size_read(devinfo, i)); |
| } |
| |
| return true; |
| } else { |
| return false; |
| } |
| } |
| |
| /** |
| * Like is_copy_payload(), but the instruction is required to source data from |
| * at least two disjoint VGRFs. |
| * |
| * This doesn't necessarily rule out the elimination of this instruction |
| * through register coalescing, but due to limitations of the register |
| * coalesce pass it might be impossible to do so directly until a later stage, |
| * when the LOAD_PAYLOAD instruction is unrolled into a sequence of MOV |
| * instructions. |
| */ |
| inline bool |
| is_multi_copy_payload(const struct intel_device_info *devinfo, |
| const fs_inst *inst) |
| { |
| if (is_copy_payload(devinfo, VGRF, inst)) { |
| for (unsigned i = 0; i < inst->sources; i++) { |
| if (inst->src[i].nr != inst->src[0].nr) |
| return true; |
| } |
| } |
| |
| return false; |
| } |
| |
| /** |
| * Like is_identity_payload(), but the instruction is required to copy the |
| * whole contents of a single VGRF into the destination. |
| * |
| * This means that there is a good chance that the instruction will be |
| * eliminated through register coalescing, but it's neither a necessary nor a |
| * sufficient condition for that to happen -- E.g. consider the case where |
| * source and destination registers diverge due to other instructions in the |
| * program overwriting part of their contents, which isn't something we can |
| * predict up front based on a cheap strictly local test of the copy |
| * instruction. |
| */ |
| inline bool |
| is_coalescing_payload(const struct intel_device_info *devinfo, |
| const brw::simple_allocator &alloc, const fs_inst *inst) |
| { |
| return is_identity_payload(devinfo, VGRF, inst) && |
| inst->src[0].offset == 0 && |
| alloc.sizes[inst->src[0].nr] * REG_SIZE == inst->size_written; |
| } |
| |
| bool |
| has_bank_conflict(const struct brw_isa_info *isa, const fs_inst *inst); |
| |
| /* Return the subset of flag registers that an instruction could |
| * potentially read or write based on the execution controls and flag |
| * subregister number of the instruction. |
| */ |
| static inline unsigned |
| brw_fs_flag_mask(const fs_inst *inst, unsigned width) |
| { |
| assert(util_is_power_of_two_nonzero(width)); |
| const unsigned start = (inst->flag_subreg * 16 + inst->group) & |
| ~(width - 1); |
| const unsigned end = start + ALIGN(inst->exec_size, width); |
| return ((1 << DIV_ROUND_UP(end, 8)) - 1) & ~((1 << (start / 8)) - 1); |
| } |
| |
| static inline unsigned |
| brw_fs_bit_mask(unsigned n) |
| { |
| return (n >= CHAR_BIT * sizeof(brw_fs_bit_mask(n)) ? ~0u : (1u << n) - 1); |
| } |
| |
| static inline unsigned |
| brw_fs_flag_mask(const brw_reg &r, unsigned sz) |
| { |
| if (r.file == ARF) { |
| const unsigned start = (r.nr - BRW_ARF_FLAG) * 4 + r.subnr; |
| const unsigned end = start + sz; |
| return brw_fs_bit_mask(end) & ~brw_fs_bit_mask(start); |
| } else { |
| return 0; |
| } |
| } |
