xref: /netbsd-src/external/gpl3/gcc/dist/gcc/cse.cc (revision b1e838363e3c6fc78a55519254d99869742dd33c)

/* Common subexpression elimination for GNU compiler.
   Copyright (C) 1987-2022 Free Software Foundation, Inc.

This file is part of GCC.

GCC is free software; you can redistribute it and/or modify it under
the terms of the GNU General Public License as published by the Free
Software Foundation; either version 3, or (at your option) any later
version.

GCC is distributed in the hope that it will be useful, but WITHOUT ANY
WARRANTY; without even the implied warranty of MERCHANTABILITY or
FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
for more details.

You should have received a copy of the GNU General Public License
along with GCC; see the file COPYING3.  If not see
<http://www.gnu.org/licenses/>.  */

#include "config.h"
#include "system.h"
#include "coretypes.h"
#include "backend.h"
#include "target.h"
#include "rtl.h"
#include "tree.h"
#include "cfghooks.h"
#include "df.h"
#include "memmodel.h"
#include "tm_p.h"
#include "insn-config.h"
#include "regs.h"
#include "emit-rtl.h"
#include "recog.h"
#include "cfgrtl.h"
#include "cfganal.h"
#include "cfgcleanup.h"
#include "alias.h"
#include "toplev.h"
#include "rtlhooks-def.h"
#include "tree-pass.h"
#include "dbgcnt.h"
#include "rtl-iter.h"
#include "regs.h"
#include "function-abi.h"
#include "rtlanal.h"
#include "expr.h"

/* The basic idea of common subexpression elimination is to go
   through the code, keeping a record of expressions that would
   have the same value at the current scan point, and replacing
   expressions encountered with the cheapest equivalent expression.

   It is too complicated to keep track of the different possibilities
   when control paths merge in this code; so, at each label, we forget all
   that is known and start fresh.  This can be described as processing each
   extended basic block separately.  We have a separate pass to perform
   global CSE.

   Note CSE can turn a conditional or computed jump into a nop or
   an unconditional jump.  When this occurs we arrange to run the jump
   optimizer after CSE to delete the unreachable code.

   We use two data structures to record the equivalent expressions:
   a hash table for most expressions, and a vector of "quantity
   numbers" to record equivalent (pseudo) registers.

   The use of the special data structure for registers is desirable
   because it is faster.  It is possible because registers references
   contain a fairly small number, the register number, taken from
   a contiguously allocated series, and two register references are
   identical if they have the same number.  General expressions
   do not have any such thing, so the only way to retrieve the
   information recorded on an expression other than a register
   is to keep it in a hash table.

Registers and "quantity numbers":

   At the start of each basic block, all of the (hardware and pseudo)
   registers used in the function are given distinct quantity
   numbers to indicate their contents.  During scan, when the code
   copies one register into another, we copy the quantity number.
   When a register is loaded in any other way, we allocate a new
   quantity number to describe the value generated by this operation.
   `REG_QTY (N)' records what quantity register N is currently thought
   of as containing.

   All real quantity numbers are greater than or equal to zero.
   If register N has not been assigned a quantity, `REG_QTY (N)' will
   equal -N - 1, which is always negative.

   Quantity numbers below zero do not exist and none of the `qty_table'
   entries should be referenced with a negative index.

   We also maintain a bidirectional chain of registers for each
   quantity number.  The `qty_table` members `first_reg' and `last_reg',
   and `reg_eqv_table' members `next' and `prev' hold these chains.

   The first register in a chain is the one whose lifespan is least local.
   Among equals, it is the one that was seen first.
   We replace any equivalent register with that one.

   If two registers have the same quantity number, it must be true that
   REG expressions with qty_table `mode' must be in the hash table for both
   registers and must be in the same class.

   The converse is not true.  Since hard registers may be referenced in
   any mode, two REG expressions might be equivalent in the hash table
   but not have the same quantity number if the quantity number of one
   of the registers is not the same mode as those expressions.

Constants and quantity numbers

   When a quantity has a known constant value, that value is stored
   in the appropriate qty_table `const_rtx'.  This is in addition to
   putting the constant in the hash table as is usual for non-regs.

   Whether a reg or a constant is preferred is determined by the configuration
   macro CONST_COSTS and will often depend on the constant value.  In any
   event, expressions containing constants can be simplified, by fold_rtx.

   When a quantity has a known nearly constant value (such as an address
   of a stack slot), that value is stored in the appropriate qty_table
   `const_rtx'.

   Integer constants don't have a machine mode.  However, cse
   determines the intended machine mode from the destination
   of the instruction that moves the constant.  The machine mode
   is recorded in the hash table along with the actual RTL
   constant expression so that different modes are kept separate.

Other expressions:

   To record known equivalences among expressions in general
   we use a hash table called `table'.  It has a fixed number of buckets
   that contain chains of `struct table_elt' elements for expressions.
   These chains connect the elements whose expressions have the same
   hash codes.

   Other chains through the same elements connect the elements which
   currently have equivalent values.

   Register references in an expression are canonicalized before hashing
   the expression.  This is done using `reg_qty' and qty_table `first_reg'.
   The hash code of a register reference is computed using the quantity
   number, not the register number.

   When the value of an expression changes, it is necessary to remove from the
   hash table not just that expression but all expressions whose values
   could be different as a result.

     1. If the value changing is in memory, except in special cases
     ANYTHING referring to memory could be changed.  That is because
     nobody knows where a pointer does not point.
     The function `invalidate_memory' removes what is necessary.

     The special cases are when the address is constant or is
     a constant plus a fixed register such as the frame pointer
     or a static chain pointer.  When such addresses are stored in,
     we can tell exactly which other such addresses must be invalidated
     due to overlap.  `invalidate' does this.
     All expressions that refer to non-constant
     memory addresses are also invalidated.  `invalidate_memory' does this.

     2. If the value changing is a register, all expressions
     containing references to that register, and only those,
     must be removed.

   Because searching the entire hash table for expressions that contain
   a register is very slow, we try to figure out when it isn't necessary.
   Precisely, this is necessary only when expressions have been
   entered in the hash table using this register, and then the value has
   changed, and then another expression wants to be added to refer to
   the register's new value.  This sequence of circumstances is rare
   within any one basic block.

   `REG_TICK' and `REG_IN_TABLE', accessors for members of
   cse_reg_info, are used to detect this case.  REG_TICK (i) is
   incremented whenever a value is stored in register i.
   REG_IN_TABLE (i) holds -1 if no references to register i have been
   entered in the table; otherwise, it contains the value REG_TICK (i)
   had when the references were entered.  If we want to enter a
   reference and REG_IN_TABLE (i) != REG_TICK (i), we must scan and
   remove old references.  Until we want to enter a new entry, the
   mere fact that the two vectors don't match makes the entries be
   ignored if anyone tries to match them.

   Registers themselves are entered in the hash table as well as in
   the equivalent-register chains.  However, `REG_TICK' and
   `REG_IN_TABLE' do not apply to expressions which are simple
   register references.  These expressions are removed from the table
   immediately when they become invalid, and this can be done even if
   we do not immediately search for all the expressions that refer to
   the register.

   A CLOBBER rtx in an instruction invalidates its operand for further
   reuse.  A CLOBBER or SET rtx whose operand is a MEM:BLK
   invalidates everything that resides in memory.

Related expressions:

   Constant expressions that differ only by an additive integer
   are called related.  When a constant expression is put in
   the table, the related expression with no constant term
   is also entered.  These are made to point at each other
   so that it is possible to find out if there exists any
   register equivalent to an expression related to a given expression.  */

/* Length of qty_table vector.  We know in advance we will not need
   a quantity number this big.  */

static int max_qty;

/* Next quantity number to be allocated.
   This is 1 + the largest number needed so far.  */

static int next_qty;

/* Per-qty information tracking.

   `first_reg' and `last_reg' track the head and tail of the
   chain of registers which currently contain this quantity.

   `mode' contains the machine mode of this quantity.

   `const_rtx' holds the rtx of the constant value of this
   quantity, if known.  A summations of the frame/arg pointer
   and a constant can also be entered here.  When this holds
   a known value, `const_insn' is the insn which stored the
   constant value.

   `comparison_{code,const,qty}' are used to track when a
   comparison between a quantity and some constant or register has
   been passed.  In such a case, we know the results of the comparison
   in case we see it again.  These members record a comparison that
   is known to be true.  `comparison_code' holds the rtx code of such
   a comparison, else it is set to UNKNOWN and the other two
   comparison members are undefined.  `comparison_const' holds
   the constant being compared against, or zero if the comparison
   is not against a constant.  `comparison_qty' holds the quantity
   being compared against when the result is known.  If the comparison
   is not with a register, `comparison_qty' is -1.  */

struct qty_table_elem
{
  rtx const_rtx;
  rtx_insn *const_insn;
  rtx comparison_const;
  int comparison_qty;
  unsigned int first_reg, last_reg;
  /* The sizes of these fields should match the sizes of the
     code and mode fields of struct rtx_def (see rtl.h).  */
  ENUM_BITFIELD(rtx_code) comparison_code : 16;
  ENUM_BITFIELD(machine_mode) mode : 8;
};

/* The table of all qtys, indexed by qty number.  */
static struct qty_table_elem *qty_table;

/* Insn being scanned.  */

static rtx_insn *this_insn;
static bool optimize_this_for_speed_p;

/* Index by register number, gives the number of the next (or
   previous) register in the chain of registers sharing the same
   value.

   Or -1 if this register is at the end of the chain.

   If REG_QTY (N) == -N - 1, reg_eqv_table[N].next is undefined.  */

/* Per-register equivalence chain.  */
struct reg_eqv_elem
{
  int next, prev;
};

/* The table of all register equivalence chains.  */
static struct reg_eqv_elem *reg_eqv_table;

struct cse_reg_info
{
  /* The timestamp at which this register is initialized.  */
  unsigned int timestamp;

  /* The quantity number of the register's current contents.  */
  int reg_qty;

  /* The number of times the register has been altered in the current
     basic block.  */
  int reg_tick;

  /* The REG_TICK value at which rtx's containing this register are
     valid in the hash table.  If this does not equal the current
     reg_tick value, such expressions existing in the hash table are
     invalid.  */
  int reg_in_table;

  /* The SUBREG that was set when REG_TICK was last incremented.  Set
     to -1 if the last store was to the whole register, not a subreg.  */
  unsigned int subreg_ticked;
};

/* A table of cse_reg_info indexed by register numbers.  */
static struct cse_reg_info *cse_reg_info_table;

/* The size of the above table.  */
static unsigned int cse_reg_info_table_size;

/* The index of the first entry that has not been initialized.  */
static unsigned int cse_reg_info_table_first_uninitialized;

/* The timestamp at the beginning of the current run of
   cse_extended_basic_block.  We increment this variable at the beginning of
   the current run of cse_extended_basic_block.  The timestamp field of a
   cse_reg_info entry matches the value of this variable if and only
   if the entry has been initialized during the current run of
   cse_extended_basic_block.  */
static unsigned int cse_reg_info_timestamp;

/* A HARD_REG_SET containing all the hard registers for which there is
   currently a REG expression in the hash table.  Note the difference
   from the above variables, which indicate if the REG is mentioned in some
   expression in the table.  */

static HARD_REG_SET hard_regs_in_table;

/* True if CSE has altered the CFG.  */
static bool cse_cfg_altered;

/* True if CSE has altered conditional jump insns in such a way
   that jump optimization should be redone.  */
static bool cse_jumps_altered;

/* True if we put a LABEL_REF into the hash table for an INSN
   without a REG_LABEL_OPERAND, we have to rerun jump after CSE
   to put in the note.  */
static bool recorded_label_ref;

/* canon_hash stores 1 in do_not_record if it notices a reference to PC or
   some other volatile subexpression.  */

static int do_not_record;

/* canon_hash stores 1 in hash_arg_in_memory
   if it notices a reference to memory within the expression being hashed.  */

static int hash_arg_in_memory;

/* The hash table contains buckets which are chains of `struct table_elt's,
   each recording one expression's information.
   That expression is in the `exp' field.

   The canon_exp field contains a canonical (from the point of view of
   alias analysis) version of the `exp' field.

   Those elements with the same hash code are chained in both directions
   through the `next_same_hash' and `prev_same_hash' fields.

   Each set of expressions with equivalent values
   are on a two-way chain through the `next_same_value'
   and `prev_same_value' fields, and all point with
   the `first_same_value' field at the first element in
   that chain.  The chain is in order of increasing cost.
   Each element's cost value is in its `cost' field.

   The `in_memory' field is nonzero for elements that
   involve any reference to memory.  These elements are removed
   whenever a write is done to an unidentified location in memory.
   To be safe, we assume that a memory address is unidentified unless
   the address is either a symbol constant or a constant plus
   the frame pointer or argument pointer.

   The `related_value' field is used to connect related expressions
   (that differ by adding an integer).
   The related expressions are chained in a circular fashion.
   `related_value' is zero for expressions for which this
   chain is not useful.

   The `cost' field stores the cost of this element's expression.
   The `regcost' field stores the value returned by approx_reg_cost for
   this element's expression.

   The `is_const' flag is set if the element is a constant (including
   a fixed address).

   The `flag' field is used as a temporary during some search routines.

   The `mode' field is usually the same as GET_MODE (`exp'), but
   if `exp' is a CONST_INT and has no machine mode then the `mode'
   field is the mode it was being used as.  Each constant is
   recorded separately for each mode it is used with.  */

struct table_elt
{
  rtx exp;
  rtx canon_exp;
  struct table_elt *next_same_hash;
  struct table_elt *prev_same_hash;
  struct table_elt *next_same_value;
  struct table_elt *prev_same_value;
  struct table_elt *first_same_value;
  struct table_elt *related_value;
  int cost;
  int regcost;
  /* The size of this field should match the size
     of the mode field of struct rtx_def (see rtl.h).  */
  ENUM_BITFIELD(machine_mode) mode : 8;
  char in_memory;
  char is_const;
  char flag;
};

/* We don't want a lot of buckets, because we rarely have very many
   things stored in the hash table, and a lot of buckets slows
   down a lot of loops that happen frequently.  */
#define HASH_SHIFT	5
#define HASH_SIZE	(1 << HASH_SHIFT)
#define HASH_MASK	(HASH_SIZE - 1)

/* Compute hash code of X in mode M.  Special-case case where X is a pseudo
   register (hard registers may require `do_not_record' to be set).  */

#define HASH(X, M)	\
 ((REG_P (X) && REGNO (X) >= FIRST_PSEUDO_REGISTER	\
  ? (((unsigned) REG << 7) + (unsigned) REG_QTY (REGNO (X)))	\
  : canon_hash (X, M)) & HASH_MASK)

/* Like HASH, but without side-effects.  */
#define SAFE_HASH(X, M)	\
 ((REG_P (X) && REGNO (X) >= FIRST_PSEUDO_REGISTER	\
  ? (((unsigned) REG << 7) + (unsigned) REG_QTY (REGNO (X)))	\
  : safe_hash (X, M)) & HASH_MASK)

/* Determine whether register number N is considered a fixed register for the
   purpose of approximating register costs.
   It is desirable to replace other regs with fixed regs, to reduce need for
   non-fixed hard regs.
   A reg wins if it is either the frame pointer or designated as fixed.  */
#define FIXED_REGNO_P(N)  \
  ((N) == FRAME_POINTER_REGNUM || (N) == HARD_FRAME_POINTER_REGNUM \
   || fixed_regs[N] || global_regs[N])

/* Compute cost of X, as stored in the `cost' field of a table_elt.  Fixed
   hard registers and pointers into the frame are the cheapest with a cost
   of 0.  Next come pseudos with a cost of one and other hard registers with
   a cost of 2.  Aside from these special cases, call `rtx_cost'.  */

#define CHEAP_REGNO(N)							\
  (REGNO_PTR_FRAME_P (N)						\
   || (HARD_REGISTER_NUM_P (N)						\
       && FIXED_REGNO_P (N) && REGNO_REG_CLASS (N) != NO_REGS))

#define COST(X, MODE)							\
  (REG_P (X) ? 0 : notreg_cost (X, MODE, SET, 1))
#define COST_IN(X, MODE, OUTER, OPNO)					\
  (REG_P (X) ? 0 : notreg_cost (X, MODE, OUTER, OPNO))

/* Get the number of times this register has been updated in this
   basic block.  */

#define REG_TICK(N) (get_cse_reg_info (N)->reg_tick)

/* Get the point at which REG was recorded in the table.  */

#define REG_IN_TABLE(N) (get_cse_reg_info (N)->reg_in_table)

/* Get the SUBREG set at the last increment to REG_TICK (-1 if not a
   SUBREG).  */

#define SUBREG_TICKED(N) (get_cse_reg_info (N)->subreg_ticked)

/* Get the quantity number for REG.  */

#define REG_QTY(N) (get_cse_reg_info (N)->reg_qty)

/* Determine if the quantity number for register X represents a valid index
   into the qty_table.  */

#define REGNO_QTY_VALID_P(N) (REG_QTY (N) >= 0)

/* Compare table_elt X and Y and return true iff X is cheaper than Y.  */

#define CHEAPER(X, Y) \
 (preferable ((X)->cost, (X)->regcost, (Y)->cost, (Y)->regcost) < 0)

static struct table_elt *table[HASH_SIZE];

/* Chain of `struct table_elt's made so far for this function
   but currently removed from the table.  */

static struct table_elt *free_element_chain;

/* Trace a patch through the CFG.  */

struct branch_path
{
  /* The basic block for this path entry.  */
  basic_block bb;
};

/* This data describes a block that will be processed by
   cse_extended_basic_block.  */

struct cse_basic_block_data
{
  /* Total number of SETs in block.  */
  int nsets;
  /* Size of current branch path, if any.  */
  int path_size;
  /* Current path, indicating which basic_blocks will be processed.  */
  struct branch_path *path;
};


/* Pointers to the live in/live out bitmaps for the boundaries of the
   current EBB.  */
static bitmap cse_ebb_live_in, cse_ebb_live_out;

/* A simple bitmap to track which basic blocks have been visited
   already as part of an already processed extended basic block.  */
static sbitmap cse_visited_basic_blocks;

static bool fixed_base_plus_p (rtx x);
static int notreg_cost (rtx, machine_mode, enum rtx_code, int);
static int preferable (int, int, int, int);
static void new_basic_block (void);
static void make_new_qty (unsigned int, machine_mode);
static void make_regs_eqv (unsigned int, unsigned int);
static void delete_reg_equiv (unsigned int);
static int mention_regs (rtx);
static int insert_regs (rtx, struct table_elt *, int);
static void remove_from_table (struct table_elt *, unsigned);
static void remove_pseudo_from_table (rtx, unsigned);
static struct table_elt *lookup (rtx, unsigned, machine_mode);
static struct table_elt *lookup_for_remove (rtx, unsigned, machine_mode);
static rtx lookup_as_function (rtx, enum rtx_code);
static struct table_elt *insert_with_costs (rtx, struct table_elt *, unsigned,
					    machine_mode, int, int);
static struct table_elt *insert (rtx, struct table_elt *, unsigned,
				 machine_mode);
static void merge_equiv_classes (struct table_elt *, struct table_elt *);
static void invalidate (rtx, machine_mode);
static void remove_invalid_refs (unsigned int);
static void remove_invalid_subreg_refs (unsigned int, poly_uint64,
					machine_mode);
static void rehash_using_reg (rtx);
static void invalidate_memory (void);
static rtx use_related_value (rtx, struct table_elt *);

static inline unsigned canon_hash (rtx, machine_mode);
static inline unsigned safe_hash (rtx, machine_mode);
static inline unsigned hash_rtx_string (const char *);

static rtx canon_reg (rtx, rtx_insn *);
static enum rtx_code find_comparison_args (enum rtx_code, rtx *, rtx *,
					   machine_mode *,
					   machine_mode *);
static rtx fold_rtx (rtx, rtx_insn *);
static rtx equiv_constant (rtx);
static void record_jump_equiv (rtx_insn *, bool);
static void record_jump_cond (enum rtx_code, machine_mode, rtx, rtx,
			      int);
static void cse_insn (rtx_insn *);
static void cse_prescan_path (struct cse_basic_block_data *);
static void invalidate_from_clobbers (rtx_insn *);
static void invalidate_from_sets_and_clobbers (rtx_insn *);
static void cse_extended_basic_block (struct cse_basic_block_data *);
extern void dump_class (struct table_elt*);
static void get_cse_reg_info_1 (unsigned int regno);
static struct cse_reg_info * get_cse_reg_info (unsigned int regno);

static void flush_hash_table (void);
static bool insn_live_p (rtx_insn *, int *);
static bool set_live_p (rtx, int *);
static void cse_change_cc_mode_insn (rtx_insn *, rtx);
static void cse_change_cc_mode_insns (rtx_insn *, rtx_insn *, rtx);
static machine_mode cse_cc_succs (basic_block, basic_block, rtx, rtx,
				       bool);


#undef RTL_HOOKS_GEN_LOWPART
#define RTL_HOOKS_GEN_LOWPART		gen_lowpart_if_possible

static const struct rtl_hooks cse_rtl_hooks = RTL_HOOKS_INITIALIZER;

/* Nonzero if X has the form (PLUS frame-pointer integer).  */

static bool
fixed_base_plus_p (rtx x)
{
  switch (GET_CODE (x))
    {
    case REG:
      if (x == frame_pointer_rtx || x == hard_frame_pointer_rtx)
	return true;
      if (x == arg_pointer_rtx && fixed_regs[ARG_POINTER_REGNUM])
	return true;
      return false;

    case PLUS:
      if (!CONST_INT_P (XEXP (x, 1)))
	return false;
      return fixed_base_plus_p (XEXP (x, 0));

    default:
      return false;
    }
}

/* Dump the expressions in the equivalence class indicated by CLASSP.
   This function is used only for debugging.  */
DEBUG_FUNCTION void
dump_class (struct table_elt *classp)
{
  struct table_elt *elt;

  fprintf (stderr, "Equivalence chain for ");
  print_rtl (stderr, classp->exp);
  fprintf (stderr, ": \n");

  for (elt = classp->first_same_value; elt; elt = elt->next_same_value)
    {
      print_rtl (stderr, elt->exp);
      fprintf (stderr, "\n");
    }
}

/* Return an estimate of the cost of the registers used in an rtx.
   This is mostly the number of different REG expressions in the rtx;
   however for some exceptions like fixed registers we use a cost of
   0.  If any other hard register reference occurs, return MAX_COST.  */

static int
approx_reg_cost (const_rtx x)
{
  int cost = 0;
  subrtx_iterator::array_type array;
  FOR_EACH_SUBRTX (iter, array, x, NONCONST)
    {
      const_rtx x = *iter;
      if (REG_P (x))
	{
	  unsigned int regno = REGNO (x);
	  if (!CHEAP_REGNO (regno))
	    {
	      if (regno < FIRST_PSEUDO_REGISTER)
		{
		  if (targetm.small_register_classes_for_mode_p (GET_MODE (x)))
		    return MAX_COST;
		  cost += 2;
		}
	      else
		cost += 1;
	    }
	}
    }
  return cost;
}

/* Return a negative value if an rtx A, whose costs are given by COST_A
   and REGCOST_A, is more desirable than an rtx B.
   Return a positive value if A is less desirable, or 0 if the two are
   equally good.  */
static int
preferable (int cost_a, int regcost_a, int cost_b, int regcost_b)
{
  /* First, get rid of cases involving expressions that are entirely
     unwanted.  */
  if (cost_a != cost_b)
    {
      if (cost_a == MAX_COST)
	return 1;
      if (cost_b == MAX_COST)
	return -1;
    }

  /* Avoid extending lifetimes of hardregs.  */
  if (regcost_a != regcost_b)
    {
      if (regcost_a == MAX_COST)
	return 1;
      if (regcost_b == MAX_COST)
	return -1;
    }

  /* Normal operation costs take precedence.  */
  if (cost_a != cost_b)
    return cost_a - cost_b;
  /* Only if these are identical consider effects on register pressure.  */
  if (regcost_a != regcost_b)
    return regcost_a - regcost_b;
  return 0;
}

/* Internal function, to compute cost when X is not a register; called
   from COST macro to keep it simple.  */

static int
notreg_cost (rtx x, machine_mode mode, enum rtx_code outer, int opno)
{
  scalar_int_mode int_mode, inner_mode;
  return ((GET_CODE (x) == SUBREG
	   && REG_P (SUBREG_REG (x))
	   && is_int_mode (mode, &int_mode)
	   && is_int_mode (GET_MODE (SUBREG_REG (x)), &inner_mode)
	   && GET_MODE_SIZE (int_mode) < GET_MODE_SIZE (inner_mode)
	   && subreg_lowpart_p (x)
	   && TRULY_NOOP_TRUNCATION_MODES_P (int_mode, inner_mode))
	  ? 0
	  : rtx_cost (x, mode, outer, opno, optimize_this_for_speed_p) * 2);
}


/* Initialize CSE_REG_INFO_TABLE.  */

static void
init_cse_reg_info (unsigned int nregs)
{
  /* Do we need to grow the table?  */
  if (nregs > cse_reg_info_table_size)
    {
      unsigned int new_size;

      if (cse_reg_info_table_size < 2048)
	{
	  /* Compute a new size that is a power of 2 and no smaller
	     than the large of NREGS and 64.  */
	  new_size = (cse_reg_info_table_size
		      ? cse_reg_info_table_size : 64);

	  while (new_size < nregs)
	    new_size *= 2;
	}
      else
	{
	  /* If we need a big table, allocate just enough to hold
	     NREGS registers.  */
	  new_size = nregs;
	}

      /* Reallocate the table with NEW_SIZE entries.  */
      free (cse_reg_info_table);
      cse_reg_info_table = XNEWVEC (struct cse_reg_info, new_size);
      cse_reg_info_table_size = new_size;
      cse_reg_info_table_first_uninitialized = 0;
    }

  /* Do we have all of the first NREGS entries initialized?  */
  if (cse_reg_info_table_first_uninitialized < nregs)
    {
      unsigned int old_timestamp = cse_reg_info_timestamp - 1;
      unsigned int i;

      /* Put the old timestamp on newly allocated entries so that they
	 will all be considered out of date.  We do not touch those
	 entries beyond the first NREGS entries to be nice to the
	 virtual memory.  */
      for (i = cse_reg_info_table_first_uninitialized; i < nregs; i++)
	cse_reg_info_table[i].timestamp = old_timestamp;

      cse_reg_info_table_first_uninitialized = nregs;
    }
}

/* Given REGNO, initialize the cse_reg_info entry for REGNO.  */

static void
get_cse_reg_info_1 (unsigned int regno)
{
  /* Set TIMESTAMP field to CSE_REG_INFO_TIMESTAMP so that this
     entry will be considered to have been initialized.  */
  cse_reg_info_table[regno].timestamp = cse_reg_info_timestamp;

  /* Initialize the rest of the entry.  */
  cse_reg_info_table[regno].reg_tick = 1;
  cse_reg_info_table[regno].reg_in_table = -1;
  cse_reg_info_table[regno].subreg_ticked = -1;
  cse_reg_info_table[regno].reg_qty = -regno - 1;
}

/* Find a cse_reg_info entry for REGNO.  */

static inline struct cse_reg_info *
get_cse_reg_info (unsigned int regno)
{
  struct cse_reg_info *p = &cse_reg_info_table[regno];

  /* If this entry has not been initialized, go ahead and initialize
     it.  */
  if (p->timestamp != cse_reg_info_timestamp)
    get_cse_reg_info_1 (regno);

  return p;
}

/* Clear the hash table and initialize each register with its own quantity,
   for a new basic block.  */

static void
new_basic_block (void)
{
  int i;

  next_qty = 0;

  /* Invalidate cse_reg_info_table.  */
  cse_reg_info_timestamp++;

  /* Clear out hash table state for this pass.  */
  CLEAR_HARD_REG_SET (hard_regs_in_table);

  /* The per-quantity values used to be initialized here, but it is
     much faster to initialize each as it is made in `make_new_qty'.  */

  for (i = 0; i < HASH_SIZE; i++)
    {
      struct table_elt *first;

      first = table[i];
      if (first != NULL)
	{
	  struct table_elt *last = first;

	  table[i] = NULL;

	  while (last->next_same_hash != NULL)
	    last = last->next_same_hash;

	  /* Now relink this hash entire chain into
	     the free element list.  */

	  last->next_same_hash = free_element_chain;
	  free_element_chain = first;
	}
    }
}

/* Say that register REG contains a quantity in mode MODE not in any
   register before and initialize that quantity.  */

static void
make_new_qty (unsigned int reg, machine_mode mode)
{
  int q;
  struct qty_table_elem *ent;
  struct reg_eqv_elem *eqv;

  gcc_assert (next_qty < max_qty);

  q = REG_QTY (reg) = next_qty++;
  ent = &qty_table[q];
  ent->first_reg = reg;
  ent->last_reg = reg;
  ent->mode = mode;
  ent->const_rtx = ent->const_insn = NULL;
  ent->comparison_code = UNKNOWN;

  eqv = &reg_eqv_table[reg];
  eqv->next = eqv->prev = -1;
}

/* Make reg NEW equivalent to reg OLD.
   OLD is not changing; NEW is.  */

static void
make_regs_eqv (unsigned int new_reg, unsigned int old_reg)
{
  unsigned int lastr, firstr;
  int q = REG_QTY (old_reg);
  struct qty_table_elem *ent;

  ent = &qty_table[q];

  /* Nothing should become eqv until it has a "non-invalid" qty number.  */
  gcc_assert (REGNO_QTY_VALID_P (old_reg));

  REG_QTY (new_reg) = q;
  firstr = ent->first_reg;
  lastr = ent->last_reg;

  /* Prefer fixed hard registers to anything.  Prefer pseudo regs to other
     hard regs.  Among pseudos, if NEW will live longer than any other reg
     of the same qty, and that is beyond the current basic block,
     make it the new canonical replacement for this qty.  */
  if (! (firstr < FIRST_PSEUDO_REGISTER && FIXED_REGNO_P (firstr))
      /* Certain fixed registers might be of the class NO_REGS.  This means
	 that not only can they not be allocated by the compiler, but
	 they cannot be used in substitutions or canonicalizations
	 either.  */
      && (new_reg >= FIRST_PSEUDO_REGISTER || REGNO_REG_CLASS (new_reg) != NO_REGS)
      && ((new_reg < FIRST_PSEUDO_REGISTER && FIXED_REGNO_P (new_reg))
	  || (new_reg >= FIRST_PSEUDO_REGISTER
	      && (firstr < FIRST_PSEUDO_REGISTER
		  || (bitmap_bit_p (cse_ebb_live_out, new_reg)
		      && !bitmap_bit_p (cse_ebb_live_out, firstr))
		  || (bitmap_bit_p (cse_ebb_live_in, new_reg)
		      && !bitmap_bit_p (cse_ebb_live_in, firstr))))))
    {
      reg_eqv_table[firstr].prev = new_reg;
      reg_eqv_table[new_reg].next = firstr;
      reg_eqv_table[new_reg].prev = -1;
      ent->first_reg = new_reg;
    }
  else
    {
      /* If NEW is a hard reg (known to be non-fixed), insert at end.
	 Otherwise, insert before any non-fixed hard regs that are at the
	 end.  Registers of class NO_REGS cannot be used as an
	 equivalent for anything.  */
      while (lastr < FIRST_PSEUDO_REGISTER && reg_eqv_table[lastr].prev >= 0
	     && (REGNO_REG_CLASS (lastr) == NO_REGS || ! FIXED_REGNO_P (lastr))
	     && new_reg >= FIRST_PSEUDO_REGISTER)
	lastr = reg_eqv_table[lastr].prev;
      reg_eqv_table[new_reg].next = reg_eqv_table[lastr].next;
      if (reg_eqv_table[lastr].next >= 0)
	reg_eqv_table[reg_eqv_table[lastr].next].prev = new_reg;
      else
	qty_table[q].last_reg = new_reg;
      reg_eqv_table[lastr].next = new_reg;
      reg_eqv_table[new_reg].prev = lastr;
    }
}

/* Remove REG from its equivalence class.  */

static void
delete_reg_equiv (unsigned int reg)
{
  struct qty_table_elem *ent;
  int q = REG_QTY (reg);
  int p, n;

  /* If invalid, do nothing.  */
  if (! REGNO_QTY_VALID_P (reg))
    return;

  ent = &qty_table[q];

  p = reg_eqv_table[reg].prev;
  n = reg_eqv_table[reg].next;

  if (n != -1)
    reg_eqv_table[n].prev = p;
  else
    ent->last_reg = p;
  if (p != -1)
    reg_eqv_table[p].next = n;
  else
    ent->first_reg = n;

  REG_QTY (reg) = -reg - 1;
}

/* Remove any invalid expressions from the hash table
   that refer to any of the registers contained in expression X.

   Make sure that newly inserted references to those registers
   as subexpressions will be considered valid.

   mention_regs is not called when a register itself
   is being stored in the table.

   Return 1 if we have done something that may have changed the hash code
   of X.  */

static int
mention_regs (rtx x)
{
  enum rtx_code code;
  int i, j;
  const char *fmt;
  int changed = 0;

  if (x == 0)
    return 0;

  code = GET_CODE (x);
  if (code == REG)
    {
      unsigned int regno = REGNO (x);
      unsigned int endregno = END_REGNO (x);
      unsigned int i;

      for (i = regno; i < endregno; i++)
	{
	  if (REG_IN_TABLE (i) >= 0 && REG_IN_TABLE (i) != REG_TICK (i))
	    remove_invalid_refs (i);

	  REG_IN_TABLE (i) = REG_TICK (i);
	  SUBREG_TICKED (i) = -1;
	}

      return 0;
    }

  /* If this is a SUBREG, we don't want to discard other SUBREGs of the same
     pseudo if they don't use overlapping words.  We handle only pseudos
     here for simplicity.  */
  if (code == SUBREG && REG_P (SUBREG_REG (x))
      && REGNO (SUBREG_REG (x)) >= FIRST_PSEUDO_REGISTER)
    {
      unsigned int i = REGNO (SUBREG_REG (x));

      if (REG_IN_TABLE (i) >= 0 && REG_IN_TABLE (i) != REG_TICK (i))
	{
	  /* If REG_IN_TABLE (i) differs from REG_TICK (i) by one, and
	     the last store to this register really stored into this
	     subreg, then remove the memory of this subreg.
	     Otherwise, remove any memory of the entire register and
	     all its subregs from the table.  */
	  if (REG_TICK (i) - REG_IN_TABLE (i) > 1
	      || SUBREG_TICKED (i) != REGNO (SUBREG_REG (x)))
	    remove_invalid_refs (i);
	  else
	    remove_invalid_subreg_refs (i, SUBREG_BYTE (x), GET_MODE (x));
	}

      REG_IN_TABLE (i) = REG_TICK (i);
      SUBREG_TICKED (i) = REGNO (SUBREG_REG (x));
      return 0;
    }

  /* If X is a comparison or a COMPARE and either operand is a register
     that does not have a quantity, give it one.  This is so that a later
     call to record_jump_equiv won't cause X to be assigned a different
     hash code and not found in the table after that call.

     It is not necessary to do this here, since rehash_using_reg can
     fix up the table later, but doing this here eliminates the need to
     call that expensive function in the most common case where the only
     use of the register is in the comparison.  */

  if (code == COMPARE || COMPARISON_P (x))
    {
      if (REG_P (XEXP (x, 0))
	  && ! REGNO_QTY_VALID_P (REGNO (XEXP (x, 0))))
	if (insert_regs (XEXP (x, 0), NULL, 0))
	  {
	    rehash_using_reg (XEXP (x, 0));
	    changed = 1;
	  }

      if (REG_P (XEXP (x, 1))
	  && ! REGNO_QTY_VALID_P (REGNO (XEXP (x, 1))))
	if (insert_regs (XEXP (x, 1), NULL, 0))
	  {
	    rehash_using_reg (XEXP (x, 1));
	    changed = 1;
	  }
    }

  fmt = GET_RTX_FORMAT (code);
  for (i = GET_RTX_LENGTH (code) - 1; i >= 0; i--)
    if (fmt[i] == 'e')
      changed |= mention_regs (XEXP (x, i));
    else if (fmt[i] == 'E')
      for (j = 0; j < XVECLEN (x, i); j++)
	changed |= mention_regs (XVECEXP (x, i, j));

  return changed;
}

/* Update the register quantities for inserting X into the hash table
   with a value equivalent to CLASSP.
   (If the class does not contain a REG, it is irrelevant.)
   If MODIFIED is nonzero, X is a destination; it is being modified.
   Note that delete_reg_equiv should be called on a register
   before insert_regs is done on that register with MODIFIED != 0.

   Nonzero value means that elements of reg_qty have changed
   so X's hash code may be different.  */

static int
insert_regs (rtx x, struct table_elt *classp, int modified)
{
  if (REG_P (x))
    {
      unsigned int regno = REGNO (x);
      int qty_valid;

      /* If REGNO is in the equivalence table already but is of the
	 wrong mode for that equivalence, don't do anything here.  */

      qty_valid = REGNO_QTY_VALID_P (regno);
      if (qty_valid)
	{
	  struct qty_table_elem *ent = &qty_table[REG_QTY (regno)];

	  if (ent->mode != GET_MODE (x))
	    return 0;
	}

      if (modified || ! qty_valid)
	{
	  if (classp)
	    for (classp = classp->first_same_value;
		 classp != 0;
		 classp = classp->next_same_value)
	      if (REG_P (classp->exp)
		  && GET_MODE (classp->exp) == GET_MODE (x))
		{
		  unsigned c_regno = REGNO (classp->exp);

		  gcc_assert (REGNO_QTY_VALID_P (c_regno));

		  /* Suppose that 5 is hard reg and 100 and 101 are
		     pseudos.  Consider

		     (set (reg:si 100) (reg:si 5))
		     (set (reg:si 5) (reg:si 100))
		     (set (reg:di 101) (reg:di 5))

		     We would now set REG_QTY (101) = REG_QTY (5), but the
		     entry for 5 is in SImode.  When we use this later in
		     copy propagation, we get the register in wrong mode.  */
		  if (qty_table[REG_QTY (c_regno)].mode != GET_MODE (x))
		    continue;

		  make_regs_eqv (regno, c_regno);
		  return 1;
		}

	  /* Mention_regs for a SUBREG checks if REG_TICK is exactly one larger
	     than REG_IN_TABLE to find out if there was only a single preceding
	     invalidation - for the SUBREG - or another one, which would be
	     for the full register.  However, if we find here that REG_TICK
	     indicates that the register is invalid, it means that it has
	     been invalidated in a separate operation.  The SUBREG might be used
	     now (then this is a recursive call), or we might use the full REG
	     now and a SUBREG of it later.  So bump up REG_TICK so that
	     mention_regs will do the right thing.  */
	  if (! modified
	      && REG_IN_TABLE (regno) >= 0
	      && REG_TICK (regno) == REG_IN_TABLE (regno) + 1)
	    REG_TICK (regno)++;
	  make_new_qty (regno, GET_MODE (x));
	  return 1;
	}

      return 0;
    }

  /* If X is a SUBREG, we will likely be inserting the inner register in the
     table.  If that register doesn't have an assigned quantity number at
     this point but does later, the insertion that we will be doing now will
     not be accessible because its hash code will have changed.  So assign
     a quantity number now.  */

  else if (GET_CODE (x) == SUBREG && REG_P (SUBREG_REG (x))
	   && ! REGNO_QTY_VALID_P (REGNO (SUBREG_REG (x))))
    {
      insert_regs (SUBREG_REG (x), NULL, 0);
      mention_regs (x);
      return 1;
    }
  else
    return mention_regs (x);
}


/* Compute upper and lower anchors for CST.  Also compute the offset of CST
   from these anchors/bases such that *_BASE + *_OFFS = CST.  Return false iff
   CST is equal to an anchor.  */

static bool
compute_const_anchors (rtx cst,
		       HOST_WIDE_INT *lower_base, HOST_WIDE_INT *lower_offs,
		       HOST_WIDE_INT *upper_base, HOST_WIDE_INT *upper_offs)
{
  unsigned HOST_WIDE_INT n = UINTVAL (cst);

  *lower_base = n & ~(targetm.const_anchor - 1);
  if ((unsigned HOST_WIDE_INT) *lower_base == n)
    return false;

  *upper_base = ((n + (targetm.const_anchor - 1))
		 & ~(targetm.const_anchor - 1));
  *upper_offs = n - *upper_base;
  *lower_offs = n - *lower_base;
  return true;
}

/* Insert the equivalence between ANCHOR and (REG + OFF) in mode MODE.  */

static void
insert_const_anchor (HOST_WIDE_INT anchor, rtx reg, HOST_WIDE_INT offs,
		     machine_mode mode)
{
  struct table_elt *elt;
  unsigned hash;
  rtx anchor_exp;
  rtx exp;

  anchor_exp = gen_int_mode (anchor, mode);
  hash = HASH (anchor_exp, mode);
  elt = lookup (anchor_exp, hash, mode);
  if (!elt)
    elt = insert (anchor_exp, NULL, hash, mode);

  exp = plus_constant (mode, reg, offs);
  /* REG has just been inserted and the hash codes recomputed.  */
  mention_regs (exp);
  hash = HASH (exp, mode);

  /* Use the cost of the register rather than the whole expression.  When
     looking up constant anchors we will further offset the corresponding
     expression therefore it does not make sense to prefer REGs over
     reg-immediate additions.  Prefer instead the oldest expression.  Also
     don't prefer pseudos over hard regs so that we derive constants in
     argument registers from other argument registers rather than from the
     original pseudo that was used to synthesize the constant.  */
  insert_with_costs (exp, elt, hash, mode, COST (reg, mode), 1);
}

/* The constant CST is equivalent to the register REG.  Create
   equivalences between the two anchors of CST and the corresponding
   register-offset expressions using REG.  */

static void
insert_const_anchors (rtx reg, rtx cst, machine_mode mode)
{
  HOST_WIDE_INT lower_base, lower_offs, upper_base, upper_offs;

  if (!compute_const_anchors (cst, &lower_base, &lower_offs,
			      &upper_base, &upper_offs))
      return;

  /* Ignore anchors of value 0.  Constants accessible from zero are
     simple.  */
  if (lower_base != 0)
    insert_const_anchor (lower_base, reg, -lower_offs, mode);

  if (upper_base != 0)
    insert_const_anchor (upper_base, reg, -upper_offs, mode);
}

/* We need to express ANCHOR_ELT->exp + OFFS.  Walk the equivalence list of
   ANCHOR_ELT and see if offsetting any of the entries by OFFS would create a
   valid expression.  Return the cheapest and oldest of such expressions.  In
   *OLD, return how old the resulting expression is compared to the other
   equivalent expressions.  */

static rtx
find_reg_offset_for_const (struct table_elt *anchor_elt, HOST_WIDE_INT offs,
			   unsigned *old)
{
  struct table_elt *elt;
  unsigned idx;
  struct table_elt *match_elt;
  rtx match;

  /* Find the cheapest and *oldest* expression to maximize the chance of
     reusing the same pseudo.  */

  match_elt = NULL;
  match = NULL_RTX;
  for (elt = anchor_elt->first_same_value, idx = 0;
       elt;
       elt = elt->next_same_value, idx++)
    {
      if (match_elt && CHEAPER (match_elt, elt))
	return match;

      if (REG_P (elt->exp)
	  || (GET_CODE (elt->exp) == PLUS
	      && REG_P (XEXP (elt->exp, 0))
	      && GET_CODE (XEXP (elt->exp, 1)) == CONST_INT))
	{
	  rtx x;

	  /* Ignore expressions that are no longer valid.  */
	  if (!REG_P (elt->exp) && !exp_equiv_p (elt->exp, elt->exp, 1, false))
	    continue;

	  x = plus_constant (GET_MODE (elt->exp), elt->exp, offs);
	  if (REG_P (x)
	      || (GET_CODE (x) == PLUS
		  && IN_RANGE (INTVAL (XEXP (x, 1)),
			       -targetm.const_anchor,
			       targetm.const_anchor - 1)))
	    {
	      match = x;
	      match_elt = elt;
	      *old = idx;
	    }
	}
    }

  return match;
}

/* Try to express the constant SRC_CONST using a register+offset expression
   derived from a constant anchor.  Return it if successful or NULL_RTX,
   otherwise.  */

static rtx
try_const_anchors (rtx src_const, machine_mode mode)
{
  struct table_elt *lower_elt, *upper_elt;
  HOST_WIDE_INT lower_base, lower_offs, upper_base, upper_offs;
  rtx lower_anchor_rtx, upper_anchor_rtx;
  rtx lower_exp = NULL_RTX, upper_exp = NULL_RTX;
  unsigned lower_old, upper_old;

  /* CONST_INT is used for CC modes, but we should leave those alone.  */
  if (GET_MODE_CLASS (mode) == MODE_CC)
    return NULL_RTX;

  gcc_assert (SCALAR_INT_MODE_P (mode));
  if (!compute_const_anchors (src_const, &lower_base, &lower_offs,
			      &upper_base, &upper_offs))
    return NULL_RTX;

  lower_anchor_rtx = GEN_INT (lower_base);
  upper_anchor_rtx = GEN_INT (upper_base);
  lower_elt = lookup (lower_anchor_rtx, HASH (lower_anchor_rtx, mode), mode);
  upper_elt = lookup (upper_anchor_rtx, HASH (upper_anchor_rtx, mode), mode);

  if (lower_elt)
    lower_exp = find_reg_offset_for_const (lower_elt, lower_offs, &lower_old);
  if (upper_elt)
    upper_exp = find_reg_offset_for_const (upper_elt, upper_offs, &upper_old);

  if (!lower_exp)
    return upper_exp;
  if (!upper_exp)
    return lower_exp;

  /* Return the older expression.  */
  return (upper_old > lower_old ? upper_exp : lower_exp);
}

/* Look in or update the hash table.  */

/* Remove table element ELT from use in the table.
   HASH is its hash code, made using the HASH macro.
   It's an argument because often that is known in advance
   and we save much time not recomputing it.  */

static void
remove_from_table (struct table_elt *elt, unsigned int hash)
{
  if (elt == 0)
    return;

  /* Mark this element as removed.  See cse_insn.  */
  elt->first_same_value = 0;

  /* Remove the table element from its equivalence class.  */

  {
    struct table_elt *prev = elt->prev_same_value;
    struct table_elt *next = elt->next_same_value;

    if (next)
      next->prev_same_value = prev;

    if (prev)
      prev->next_same_value = next;
    else
      {
	struct table_elt *newfirst = next;
	while (next)
	  {
	    next->first_same_value = newfirst;
	    next = next->next_same_value;
	  }
      }
  }

  /* Remove the table element from its hash bucket.  */

  {
    struct table_elt *prev = elt->prev_same_hash;
    struct table_elt *next = elt->next_same_hash;

    if (next)
      next->prev_same_hash = prev;

    if (prev)
      prev->next_same_hash = next;
    else if (table[hash] == elt)
      table[hash] = next;
    else
      {
	/* This entry is not in the proper hash bucket.  This can happen
	   when two classes were merged by `merge_equiv_classes'.  Search
	   for the hash bucket that it heads.  This happens only very
	   rarely, so the cost is acceptable.  */
	for (hash = 0; hash < HASH_SIZE; hash++)
	  if (table[hash] == elt)
	    table[hash] = next;
      }
  }

  /* Remove the table element from its related-value circular chain.  */

  if (elt->related_value != 0 && elt->related_value != elt)
    {
      struct table_elt *p = elt->related_value;

      while (p->related_value != elt)
	p = p->related_value;
      p->related_value = elt->related_value;
      if (p->related_value == p)
	p->related_value = 0;
    }

  /* Now add it to the free element chain.  */
  elt->next_same_hash = free_element_chain;
  free_element_chain = elt;
}

/* Same as above, but X is a pseudo-register.  */

static void
remove_pseudo_from_table (rtx x, unsigned int hash)
{
  struct table_elt *elt;

  /* Because a pseudo-register can be referenced in more than one
     mode, we might have to remove more than one table entry.  */
  while ((elt = lookup_for_remove (x, hash, VOIDmode)))
    remove_from_table (elt, hash);
}

/* Look up X in the hash table and return its table element,
   or 0 if X is not in the table.

   MODE is the machine-mode of X, or if X is an integer constant
   with VOIDmode then MODE is the mode with which X will be used.

   Here we are satisfied to find an expression whose tree structure
   looks like X.  */

static struct table_elt *
lookup (rtx x, unsigned int hash, machine_mode mode)
{
  struct table_elt *p;

  for (p = table[hash]; p; p = p->next_same_hash)
    if (mode == p->mode && ((x == p->exp && REG_P (x))
			    || exp_equiv_p (x, p->exp, !REG_P (x), false)))
      return p;

  return 0;
}

/* Like `lookup' but don't care whether the table element uses invalid regs.
   Also ignore discrepancies in the machine mode of a register.  */

static struct table_elt *
lookup_for_remove (rtx x, unsigned int hash, machine_mode mode)
{
  struct table_elt *p;

  if (REG_P (x))
    {
      unsigned int regno = REGNO (x);

      /* Don't check the machine mode when comparing registers;
	 invalidating (REG:SI 0) also invalidates (REG:DF 0).  */
      for (p = table[hash]; p; p = p->next_same_hash)
	if (REG_P (p->exp)
	    && REGNO (p->exp) == regno)
	  return p;
    }
  else
    {
      for (p = table[hash]; p; p = p->next_same_hash)
	if (mode == p->mode
	    && (x == p->exp || exp_equiv_p (x, p->exp, 0, false)))
	  return p;
    }

  return 0;
}

/* Look for an expression equivalent to X and with code CODE.
   If one is found, return that expression.  */

static rtx
lookup_as_function (rtx x, enum rtx_code code)
{
  struct table_elt *p
    = lookup (x, SAFE_HASH (x, VOIDmode), GET_MODE (x));

  if (p == 0)
    return 0;

  for (p = p->first_same_value; p; p = p->next_same_value)
    if (GET_CODE (p->exp) == code
	/* Make sure this is a valid entry in the table.  */
	&& exp_equiv_p (p->exp, p->exp, 1, false))
      return p->exp;

  return 0;
}

/* Insert X in the hash table, assuming HASH is its hash code and
   CLASSP is an element of the class it should go in (or 0 if a new
   class should be made).  COST is the code of X and reg_cost is the
   cost of registers in X.  It is inserted at the proper position to
   keep the class in the order cheapest first.

   MODE is the machine-mode of X, or if X is an integer constant
   with VOIDmode then MODE is the mode with which X will be used.

   For elements of equal cheapness, the most recent one
   goes in front, except that the first element in the list
   remains first unless a cheaper element is added.  The order of
   pseudo-registers does not matter, as canon_reg will be called to
   find the cheapest when a register is retrieved from the table.

   The in_memory field in the hash table element is set to 0.
   The caller must set it nonzero if appropriate.

   You should call insert_regs (X, CLASSP, MODIFY) before calling here,
   and if insert_regs returns a nonzero value
   you must then recompute its hash code before calling here.

   If necessary, update table showing constant values of quantities.  */

static struct table_elt *
insert_with_costs (rtx x, struct table_elt *classp, unsigned int hash,
		   machine_mode mode, int cost, int reg_cost)
{
  struct table_elt *elt;

  /* If X is a register and we haven't made a quantity for it,
     something is wrong.  */
  gcc_assert (!REG_P (x) || REGNO_QTY_VALID_P (REGNO (x)));

  /* If X is a hard register, show it is being put in the table.  */
  if (REG_P (x) && REGNO (x) < FIRST_PSEUDO_REGISTER)
    add_to_hard_reg_set (&hard_regs_in_table, GET_MODE (x), REGNO (x));

  /* Put an element for X into the right hash bucket.  */

  elt = free_element_chain;
  if (elt)
    free_element_chain = elt->next_same_hash;
  else
    elt = XNEW (struct table_elt);

  elt->exp = x;
  elt->canon_exp = NULL_RTX;
  elt->cost = cost;
  elt->regcost = reg_cost;
  elt->next_same_value = 0;
  elt->prev_same_value = 0;
  elt->next_same_hash = table[hash];
  elt->prev_same_hash = 0;
  elt->related_value = 0;
  elt->in_memory = 0;
  elt->mode = mode;
  elt->is_const = (CONSTANT_P (x) || fixed_base_plus_p (x));

  if (table[hash])
    table[hash]->prev_same_hash = elt;
  table[hash] = elt;

  /* Put it into the proper value-class.  */
  if (classp)
    {
      classp = classp->first_same_value;
      if (CHEAPER (elt, classp))
	/* Insert at the head of the class.  */
	{
	  struct table_elt *p;
	  elt->next_same_value = classp;
	  classp->prev_same_value = elt;
	  elt->first_same_value = elt;

	  for (p = classp; p; p = p->next_same_value)
	    p->first_same_value = elt;
	}
      else
	{
	  /* Insert not at head of the class.  */
	  /* Put it after the last element cheaper than X.  */
	  struct table_elt *p, *next;

	  for (p = classp;
	       (next = p->next_same_value) && CHEAPER (next, elt);
	       p = next)
	    ;

	  /* Put it after P and before NEXT.  */
	  elt->next_same_value = next;
	  if (next)
	    next->prev_same_value = elt;

	  elt->prev_same_value = p;
	  p->next_same_value = elt;
	  elt->first_same_value = classp;
	}
    }
  else
    elt->first_same_value = elt;

  /* If this is a constant being set equivalent to a register or a register
     being set equivalent to a constant, note the constant equivalence.

     If this is a constant, it cannot be equivalent to a different constant,
     and a constant is the only thing that can be cheaper than a register.  So
     we know the register is the head of the class (before the constant was
     inserted).

     If this is a register that is not already known equivalent to a
     constant, we must check the entire class.

     If this is a register that is already known equivalent to an insn,
     update the qtys `const_insn' to show that `this_insn' is the latest
     insn making that quantity equivalent to the constant.  */

  if (elt->is_const && classp && REG_P (classp->exp)
      && !REG_P (x))
    {
      int exp_q = REG_QTY (REGNO (classp->exp));
      struct qty_table_elem *exp_ent = &qty_table[exp_q];

      exp_ent->const_rtx = gen_lowpart (exp_ent->mode, x);
      exp_ent->const_insn = this_insn;
    }

  else if (REG_P (x)
	   && classp
	   && ! qty_table[REG_QTY (REGNO (x))].const_rtx
	   && ! elt->is_const)
    {
      struct table_elt *p;

      for (p = classp; p != 0; p = p->next_same_value)
	{
	  if (p->is_const && !REG_P (p->exp))
	    {
	      int x_q = REG_QTY (REGNO (x));
	      struct qty_table_elem *x_ent = &qty_table[x_q];

	      x_ent->const_rtx
		= gen_lowpart (GET_MODE (x), p->exp);
	      x_ent->const_insn = this_insn;
	      break;
	    }
	}
    }

  else if (REG_P (x)
	   && qty_table[REG_QTY (REGNO (x))].const_rtx
	   && GET_MODE (x) == qty_table[REG_QTY (REGNO (x))].mode)
    qty_table[REG_QTY (REGNO (x))].const_insn = this_insn;

  /* If this is a constant with symbolic value,
     and it has a term with an explicit integer value,
     link it up with related expressions.  */
  if (GET_CODE (x) == CONST)
    {
      rtx subexp = get_related_value (x);
      unsigned subhash;
      struct table_elt *subelt, *subelt_prev;

      if (subexp != 0)
	{
	  /* Get the integer-free subexpression in the hash table.  */
	  subhash = SAFE_HASH (subexp, mode);
	  subelt = lookup (subexp, subhash, mode);
	  if (subelt == 0)
	    subelt = insert (subexp, NULL, subhash, mode);
	  /* Initialize SUBELT's circular chain if it has none.  */
	  if (subelt->related_value == 0)
	    subelt->related_value = subelt;
	  /* Find the element in the circular chain that precedes SUBELT.  */
	  subelt_prev = subelt;
	  while (subelt_prev->related_value != subelt)
	    subelt_prev = subelt_prev->related_value;
	  /* Put new ELT into SUBELT's circular chain just before SUBELT.
	     This way the element that follows SUBELT is the oldest one.  */
	  elt->related_value = subelt_prev->related_value;
	  subelt_prev->related_value = elt;
	}
    }

  return elt;
}

/* Wrap insert_with_costs by passing the default costs.  */

static struct table_elt *
insert (rtx x, struct table_elt *classp, unsigned int hash,
	machine_mode mode)
{
  return insert_with_costs (x, classp, hash, mode,
			    COST (x, mode), approx_reg_cost (x));
}


/* Given two equivalence classes, CLASS1 and CLASS2, put all the entries from
   CLASS2 into CLASS1.  This is done when we have reached an insn which makes
   the two classes equivalent.

   CLASS1 will be the surviving class; CLASS2 should not be used after this
   call.

   Any invalid entries in CLASS2 will not be copied.  */

static void
merge_equiv_classes (struct table_elt *class1, struct table_elt *class2)
{
  struct table_elt *elt, *next, *new_elt;

  /* Ensure we start with the head of the classes.  */
  class1 = class1->first_same_value;
  class2 = class2->first_same_value;

  /* If they were already equal, forget it.  */
  if (class1 == class2)
    return;

  for (elt = class2; elt; elt = next)
    {
      unsigned int hash;
      rtx exp = elt->exp;
      machine_mode mode = elt->mode;

      next = elt->next_same_value;

      /* Remove old entry, make a new one in CLASS1's class.
	 Don't do this for invalid entries as we cannot find their
	 hash code (it also isn't necessary).  */
      if (REG_P (exp) || exp_equiv_p (exp, exp, 1, false))
	{
	  bool need_rehash = false;

	  hash_arg_in_memory = 0;
	  hash = HASH (exp, mode);

	  if (REG_P (exp))
	    {
	      need_rehash = REGNO_QTY_VALID_P (REGNO (exp));
	      delete_reg_equiv (REGNO (exp));
	    }

	  if (REG_P (exp) && REGNO (exp) >= FIRST_PSEUDO_REGISTER)
	    remove_pseudo_from_table (exp, hash);
	  else
	    remove_from_table (elt, hash);

	  if (insert_regs (exp, class1, 0) || need_rehash)
	    {
	      rehash_using_reg (exp);
	      hash = HASH (exp, mode);
	    }
	  new_elt = insert (exp, class1, hash, mode);
	  new_elt->in_memory = hash_arg_in_memory;
	  if (GET_CODE (exp) == ASM_OPERANDS && elt->cost == MAX_COST)
	    new_elt->cost = MAX_COST;
	}
    }
}

/* Flush the entire hash table.  */

static void
flush_hash_table (void)
{
  int i;
  struct table_elt *p;

  for (i = 0; i < HASH_SIZE; i++)
    for (p = table[i]; p; p = table[i])
      {
	/* Note that invalidate can remove elements
	   after P in the current hash chain.  */
	if (REG_P (p->exp))
	  invalidate (p->exp, VOIDmode);
	else
	  remove_from_table (p, i);
      }
}

/* Check whether an anti dependence exists between X and EXP.  MODE and
   ADDR are as for canon_anti_dependence.  */

static bool
check_dependence (const_rtx x, rtx exp, machine_mode mode, rtx addr)
{
  subrtx_iterator::array_type array;
  FOR_EACH_SUBRTX (iter, array, x, NONCONST)
    {
      const_rtx x = *iter;
      if (MEM_P (x) && canon_anti_dependence (x, true, exp, mode, addr))
	return true;
    }
  return false;
}

/* Remove from the hash table, or mark as invalid, all expressions whose
   values could be altered by storing in register X.  */

static void
invalidate_reg (rtx x)
{
  gcc_assert (GET_CODE (x) == REG);

  /* If X is a register, dependencies on its contents are recorded
     through the qty number mechanism.  Just change the qty number of
     the register, mark it as invalid for expressions that refer to it,
     and remove it itself.  */
  unsigned int regno = REGNO (x);
  unsigned int hash = HASH (x, GET_MODE (x));

  /* Remove REGNO from any quantity list it might be on and indicate
     that its value might have changed.  If it is a pseudo, remove its
     entry from the hash table.

     For a hard register, we do the first two actions above for any
     additional hard registers corresponding to X.  Then, if any of these
     registers are in the table, we must remove any REG entries that
     overlap these registers.  */

  delete_reg_equiv (regno);
  REG_TICK (regno)++;
  SUBREG_TICKED (regno) = -1;

  if (regno >= FIRST_PSEUDO_REGISTER)
    remove_pseudo_from_table (x, hash);
  else
    {
      HOST_WIDE_INT in_table = TEST_HARD_REG_BIT (hard_regs_in_table, regno);
      unsigned int endregno = END_REGNO (x);
      unsigned int rn;
      struct table_elt *p, *next;

      CLEAR_HARD_REG_BIT (hard_regs_in_table, regno);

      for (rn = regno + 1; rn < endregno; rn++)
	{
	  in_table |= TEST_HARD_REG_BIT (hard_regs_in_table, rn);
	  CLEAR_HARD_REG_BIT (hard_regs_in_table, rn);
	  delete_reg_equiv (rn);
	  REG_TICK (rn)++;
	  SUBREG_TICKED (rn) = -1;
	}

      if (in_table)
	for (hash = 0; hash < HASH_SIZE; hash++)
	  for (p = table[hash]; p; p = next)
	    {
	      next = p->next_same_hash;

	      if (!REG_P (p->exp) || REGNO (p->exp) >= FIRST_PSEUDO_REGISTER)
		continue;

	      unsigned int tregno = REGNO (p->exp);
	      unsigned int tendregno = END_REGNO (p->exp);
	      if (tendregno > regno && tregno < endregno)
		remove_from_table (p, hash);
	    }
    }
}

/* Remove from the hash table, or mark as invalid, all expressions whose
   values could be altered by storing in X.  X is a register, a subreg, or
   a memory reference with nonvarying address (because, when a memory
   reference with a varying address is stored in, all memory references are
   removed by invalidate_memory so specific invalidation is superfluous).
   FULL_MODE, if not VOIDmode, indicates that this much should be
   invalidated instead of just the amount indicated by the mode of X.  This
   is only used for bitfield stores into memory.

   A nonvarying address may be just a register or just a symbol reference,
   or it may be either of those plus a numeric offset.  */

static void
invalidate (rtx x, machine_mode full_mode)
{
  int i;
  struct table_elt *p;
  rtx addr;

  switch (GET_CODE (x))
    {
    case REG:
      invalidate_reg (x);
      return;

    case SUBREG:
      invalidate (SUBREG_REG (x), VOIDmode);
      return;

    case PARALLEL:
      for (i = XVECLEN (x, 0) - 1; i >= 0; --i)
	invalidate (XVECEXP (x, 0, i), VOIDmode);
      return;

    case EXPR_LIST:
      /* This is part of a disjoint return value; extract the location in
	 question ignoring the offset.  */
      invalidate (XEXP (x, 0), VOIDmode);
      return;

    case MEM:
      addr = canon_rtx (get_addr (XEXP (x, 0)));
      /* Calculate the canonical version of X here so that
	 true_dependence doesn't generate new RTL for X on each call.  */
      x = canon_rtx (x);

      /* Remove all hash table elements that refer to overlapping pieces of
	 memory.  */
      if (full_mode == VOIDmode)
	full_mode = GET_MODE (x);

      for (i = 0; i < HASH_SIZE; i++)
	{
	  struct table_elt *next;

	  for (p = table[i]; p; p = next)
	    {
	      next = p->next_same_hash;
	      if (p->in_memory)
		{
		  /* Just canonicalize the expression once;
		     otherwise each time we call invalidate
		     true_dependence will canonicalize the
		     expression again.  */
		  if (!p->canon_exp)
		    p->canon_exp = canon_rtx (p->exp);
		  if (check_dependence (p->canon_exp, x, full_mode, addr))
		    remove_from_table (p, i);
		}
	    }
	}
      return;

    default:
      gcc_unreachable ();
    }
}

/* Invalidate DEST.  Used when DEST is not going to be added
   into the hash table for some reason, e.g. do_not_record
   flagged on it.  */

static void
invalidate_dest (rtx dest)
{
  if (REG_P (dest)
      || GET_CODE (dest) == SUBREG
      || MEM_P (dest))
    invalidate (dest, VOIDmode);
  else if (GET_CODE (dest) == STRICT_LOW_PART
	   || GET_CODE (dest) == ZERO_EXTRACT)
    invalidate (XEXP (dest, 0), GET_MODE (dest));
}

/* Remove all expressions that refer to register REGNO,
   since they are already invalid, and we are about to
   mark that register valid again and don't want the old
   expressions to reappear as valid.  */

static void
remove_invalid_refs (unsigned int regno)
{
  unsigned int i;
  struct table_elt *p, *next;

  for (i = 0; i < HASH_SIZE; i++)
    for (p = table[i]; p; p = next)
      {
	next = p->next_same_hash;
	if (!REG_P (p->exp) && refers_to_regno_p (regno, p->exp))
	  remove_from_table (p, i);
      }
}

/* Likewise for a subreg with subreg_reg REGNO, subreg_byte OFFSET,
   and mode MODE.  */
static void
remove_invalid_subreg_refs (unsigned int regno, poly_uint64 offset,
			    machine_mode mode)
{
  unsigned int i;
  struct table_elt *p, *next;

  for (i = 0; i < HASH_SIZE; i++)
    for (p = table[i]; p; p = next)
      {
	rtx exp = p->exp;
	next = p->next_same_hash;

	if (!REG_P (exp)
	    && (GET_CODE (exp) != SUBREG
		|| !REG_P (SUBREG_REG (exp))
		|| REGNO (SUBREG_REG (exp)) != regno
		|| ranges_maybe_overlap_p (SUBREG_BYTE (exp),
					   GET_MODE_SIZE (GET_MODE (exp)),
					   offset, GET_MODE_SIZE (mode)))
	    && refers_to_regno_p (regno, p->exp))
	  remove_from_table (p, i);
      }
}

/* Recompute the hash codes of any valid entries in the hash table that
   reference X, if X is a register, or SUBREG_REG (X) if X is a SUBREG.

   This is called when we make a jump equivalence.  */

static void
rehash_using_reg (rtx x)
{
  unsigned int i;
  struct table_elt *p, *next;
  unsigned hash;

  if (GET_CODE (x) == SUBREG)
    x = SUBREG_REG (x);

  /* If X is not a register or if the register is known not to be in any
     valid entries in the table, we have no work to do.  */

  if (!REG_P (x)
      || REG_IN_TABLE (REGNO (x)) < 0
      || REG_IN_TABLE (REGNO (x)) != REG_TICK (REGNO (x)))
    return;

  /* Scan all hash chains looking for valid entries that mention X.
     If we find one and it is in the wrong hash chain, move it.  */

  for (i = 0; i < HASH_SIZE; i++)
    for (p = table[i]; p; p = next)
      {
	next = p->next_same_hash;
	if (reg_mentioned_p (x, p->exp)
	    && exp_equiv_p (p->exp, p->exp, 1, false)
	    && i != (hash = SAFE_HASH (p->exp, p->mode)))
	  {
	    if (p->next_same_hash)
	      p->next_same_hash->prev_same_hash = p->prev_same_hash;

	    if (p->prev_same_hash)
	      p->prev_same_hash->next_same_hash = p->next_same_hash;
	    else
	      table[i] = p->next_same_hash;

	    p->next_same_hash = table[hash];
	    p->prev_same_hash = 0;
	    if (table[hash])
	      table[hash]->prev_same_hash = p;
	    table[hash] = p;
	  }
      }
}

/* Remove from the hash table any expression that is a call-clobbered
   register in INSN.  Also update their TICK values.  */

static void
invalidate_for_call (rtx_insn *insn)
{
  unsigned int regno;
  unsigned hash;
  struct table_elt *p, *next;
  int in_table = 0;
  hard_reg_set_iterator hrsi;

  /* Go through all the hard registers.  For each that might be clobbered
     in call insn INSN, remove the register from quantity chains and update
     reg_tick if defined.  Also see if any of these registers is currently
     in the table.

     ??? We could be more precise for partially-clobbered registers,
     and only invalidate values that actually occupy the clobbered part
     of the registers.  It doesn't seem worth the effort though, since
     we shouldn't see this situation much before RA.  Whatever choice
     we make here has to be consistent with the table walk below,
     so any change to this test will require a change there too.  */
  HARD_REG_SET callee_clobbers
    = insn_callee_abi (insn).full_and_partial_reg_clobbers ();
  EXECUTE_IF_SET_IN_HARD_REG_SET (callee_clobbers, 0, regno, hrsi)
    {
      delete_reg_equiv (regno);
      if (REG_TICK (regno) >= 0)
	{
	  REG_TICK (regno)++;
	  SUBREG_TICKED (regno) = -1;
	}
      in_table |= (TEST_HARD_REG_BIT (hard_regs_in_table, regno) != 0);
    }

  /* In the case where we have no call-clobbered hard registers in the
     table, we are done.  Otherwise, scan the table and remove any
     entry that overlaps a call-clobbered register.  */

  if (in_table)
    for (hash = 0; hash < HASH_SIZE; hash++)
      for (p = table[hash]; p; p = next)
	{
	  next = p->next_same_hash;

	  if (!REG_P (p->exp)
	      || REGNO (p->exp) >= FIRST_PSEUDO_REGISTER)
	    continue;

	  /* This must use the same test as above rather than the
	     more accurate clobbers_reg_p.  */
	  if (overlaps_hard_reg_set_p (callee_clobbers, GET_MODE (p->exp),
				       REGNO (p->exp)))
	    remove_from_table (p, hash);
	}
}

/* Given an expression X of type CONST,
   and ELT which is its table entry (or 0 if it
   is not in the hash table),
   return an alternate expression for X as a register plus integer.
   If none can be found, return 0.  */

static rtx
use_related_value (rtx x, struct table_elt *elt)
{
  struct table_elt *relt = 0;
  struct table_elt *p, *q;
  HOST_WIDE_INT offset;

  /* First, is there anything related known?
     If we have a table element, we can tell from that.
     Otherwise, must look it up.  */

  if (elt != 0 && elt->related_value != 0)
    relt = elt;
  else if (elt == 0 && GET_CODE (x) == CONST)
    {
      rtx subexp = get_related_value (x);
      if (subexp != 0)
	relt = lookup (subexp,
		       SAFE_HASH (subexp, GET_MODE (subexp)),
		       GET_MODE (subexp));
    }

  if (relt == 0)
    return 0;

  /* Search all related table entries for one that has an
     equivalent register.  */

  p = relt;
  while (1)
    {
      /* This loop is strange in that it is executed in two different cases.
	 The first is when X is already in the table.  Then it is searching
	 the RELATED_VALUE list of X's class (RELT).  The second case is when
	 X is not in the table.  Then RELT points to a class for the related
	 value.

	 Ensure that, whatever case we are in, that we ignore classes that have
	 the same value as X.  */

      if (rtx_equal_p (x, p->exp))
	q = 0;
      else
	for (q = p->first_same_value; q; q = q->next_same_value)
	  if (REG_P (q->exp))
	    break;

      if (q)
	break;

      p = p->related_value;

      /* We went all the way around, so there is nothing to be found.
	 Alternatively, perhaps RELT was in the table for some other reason
	 and it has no related values recorded.  */
      if (p == relt || p == 0)
	break;
    }

  if (q == 0)
    return 0;

  offset = (get_integer_term (x) - get_integer_term (p->exp));
  /* Note: OFFSET may be 0 if P->xexp and X are related by commutativity.  */
  return plus_constant (q->mode, q->exp, offset);
}


/* Hash a string.  Just add its bytes up.  */
static inline unsigned
hash_rtx_string (const char *ps)
{
  unsigned hash = 0;
  const unsigned char *p = (const unsigned char *) ps;

  if (p)
    while (*p)
      hash += *p++;

  return hash;
}

/* Same as hash_rtx, but call CB on each rtx if it is not NULL.
   When the callback returns true, we continue with the new rtx.  */

unsigned
hash_rtx_cb (const_rtx x, machine_mode mode,
             int *do_not_record_p, int *hash_arg_in_memory_p,
             bool have_reg_qty, hash_rtx_callback_function cb)
{
  int i, j;
  unsigned hash = 0;
  enum rtx_code code;
  const char *fmt;
  machine_mode newmode;
  rtx newx;

  /* Used to turn recursion into iteration.  We can't rely on GCC's
     tail-recursion elimination since we need to keep accumulating values
     in HASH.  */
 repeat:
  if (x == 0)
    return hash;

  /* Invoke the callback first.  */
  if (cb != NULL
      && ((*cb) (x, mode, &newx, &newmode)))
    {
      hash += hash_rtx_cb (newx, newmode, do_not_record_p,
                           hash_arg_in_memory_p, have_reg_qty, cb);
      return hash;
    }

  code = GET_CODE (x);
  switch (code)
    {
    case REG:
      {
	unsigned int regno = REGNO (x);

	if (do_not_record_p && !reload_completed)
	  {
	    /* On some machines, we can't record any non-fixed hard register,
	       because extending its life will cause reload problems.  We
	       consider ap, fp, sp, gp to be fixed for this purpose.

	       We also consider CCmode registers to be fixed for this purpose;
	       failure to do so leads to failure to simplify 0<100 type of
	       conditionals.

	       On all machines, we can't record any global registers.
	       Nor should we record any register that is in a small
	       class, as defined by TARGET_CLASS_LIKELY_SPILLED_P.  */
	    bool record;

	    if (regno >= FIRST_PSEUDO_REGISTER)
	      record = true;
	    else if (x == frame_pointer_rtx
		     || x == hard_frame_pointer_rtx
		     || x == arg_pointer_rtx
		     || x == stack_pointer_rtx
		     || x == pic_offset_table_rtx)
	      record = true;
	    else if (global_regs[regno])
	      record = false;
	    else if (fixed_regs[regno])
	      record = true;
	    else if (GET_MODE_CLASS (GET_MODE (x)) == MODE_CC)
	      record = true;
	    else if (targetm.small_register_classes_for_mode_p (GET_MODE (x)))
	      record = false;
	    else if (targetm.class_likely_spilled_p (REGNO_REG_CLASS (regno)))
	      record = false;
	    else
	      record = true;

	    if (!record)
	      {
		*do_not_record_p = 1;
		return 0;
	      }
	  }

	hash += ((unsigned int) REG << 7);
        hash += (have_reg_qty ? (unsigned) REG_QTY (regno) : regno);
	return hash;
      }

    /* We handle SUBREG of a REG specially because the underlying
       reg changes its hash value with every value change; we don't
       want to have to forget unrelated subregs when one subreg changes.  */
    case SUBREG:
      {
	if (REG_P (SUBREG_REG (x)))
	  {
	    hash += (((unsigned int) SUBREG << 7)
		     + REGNO (SUBREG_REG (x))
		     + (constant_lower_bound (SUBREG_BYTE (x))
			/ UNITS_PER_WORD));
	    return hash;
	  }
	break;
      }

    case CONST_INT:
      hash += (((unsigned int) CONST_INT << 7) + (unsigned int) mode
               + (unsigned int) INTVAL (x));
      return hash;

    case CONST_WIDE_INT:
      for (i = 0; i < CONST_WIDE_INT_NUNITS (x); i++)
	hash += CONST_WIDE_INT_ELT (x, i);
      return hash;

    case CONST_POLY_INT:
      {
	inchash::hash h;
	h.add_int (hash);
	for (unsigned int i = 0; i < NUM_POLY_INT_COEFFS; ++i)
	  h.add_wide_int (CONST_POLY_INT_COEFFS (x)[i]);
	return h.end ();
      }

    case CONST_DOUBLE:
      /* This is like the general case, except that it only counts
	 the integers representing the constant.  */
      hash += (unsigned int) code + (unsigned int) GET_MODE (x);
      if (TARGET_SUPPORTS_WIDE_INT == 0 && GET_MODE (x) == VOIDmode)
	hash += ((unsigned int) CONST_DOUBLE_LOW (x)
		 + (unsigned int) CONST_DOUBLE_HIGH (x));
      else
	hash += real_hash (CONST_DOUBLE_REAL_VALUE (x));
      return hash;

    case CONST_FIXED:
      hash += (unsigned int) code + (unsigned int) GET_MODE (x);
      hash += fixed_hash (CONST_FIXED_VALUE (x));
      return hash;

    case CONST_VECTOR:
      {
	int units;
	rtx elt;

	units = const_vector_encoded_nelts (x);

	for (i = 0; i < units; ++i)
	  {
	    elt = CONST_VECTOR_ENCODED_ELT (x, i);
	    hash += hash_rtx_cb (elt, GET_MODE (elt),
                                 do_not_record_p, hash_arg_in_memory_p,
                                 have_reg_qty, cb);
	  }

	return hash;
      }

      /* Assume there is only one rtx object for any given label.  */
    case LABEL_REF:
      /* We don't hash on the address of the CODE_LABEL to avoid bootstrap
	 differences and differences between each stage's debugging dumps.  */
	 hash += (((unsigned int) LABEL_REF << 7)
		  + CODE_LABEL_NUMBER (label_ref_label (x)));
      return hash;

    case SYMBOL_REF:
      {
	/* Don't hash on the symbol's address to avoid bootstrap differences.
	   Different hash values may cause expressions to be recorded in
	   different orders and thus different registers to be used in the
	   final assembler.  This also avoids differences in the dump files
	   between various stages.  */
	unsigned int h = 0;
	const unsigned char *p = (const unsigned char *) XSTR (x, 0);

	while (*p)
	  h += (h << 7) + *p++; /* ??? revisit */

	hash += ((unsigned int) SYMBOL_REF << 7) + h;
	return hash;
      }

    case MEM:
      /* We don't record if marked volatile or if BLKmode since we don't
	 know the size of the move.  */
      if (do_not_record_p && (MEM_VOLATILE_P (x) || GET_MODE (x) == BLKmode))
	{
	  *do_not_record_p = 1;
	  return 0;
	}
      if (hash_arg_in_memory_p && !MEM_READONLY_P (x))
	*hash_arg_in_memory_p = 1;

      /* Now that we have already found this special case,
	 might as well speed it up as much as possible.  */
      hash += (unsigned) MEM;
      x = XEXP (x, 0);
      goto repeat;

    case USE:
      /* A USE that mentions non-volatile memory needs special
	 handling since the MEM may be BLKmode which normally
	 prevents an entry from being made.  Pure calls are
	 marked by a USE which mentions BLKmode memory.
	 See calls.cc:emit_call_1.  */
      if (MEM_P (XEXP (x, 0))
	  && ! MEM_VOLATILE_P (XEXP (x, 0)))
	{
	  hash += (unsigned) USE;
	  x = XEXP (x, 0);

	  if (hash_arg_in_memory_p && !MEM_READONLY_P (x))
	    *hash_arg_in_memory_p = 1;

	  /* Now that we have already found this special case,
	     might as well speed it up as much as possible.  */
	  hash += (unsigned) MEM;
	  x = XEXP (x, 0);
	  goto repeat;
	}
      break;

    case PRE_DEC:
    case PRE_INC:
    case POST_DEC:
    case POST_INC:
    case PRE_MODIFY:
    case POST_MODIFY:
    case PC:
    case CALL:
    case UNSPEC_VOLATILE:
      if (do_not_record_p) {
        *do_not_record_p = 1;
        return 0;
      }
      else
        return hash;
      break;

    case ASM_OPERANDS:
      if (do_not_record_p && MEM_VOLATILE_P (x))
	{
	  *do_not_record_p = 1;
	  return 0;
	}
      else
	{
	  /* We don't want to take the filename and line into account.  */
	  hash += (unsigned) code + (unsigned) GET_MODE (x)
	    + hash_rtx_string (ASM_OPERANDS_TEMPLATE (x))
	    + hash_rtx_string (ASM_OPERANDS_OUTPUT_CONSTRAINT (x))
	    + (unsigned) ASM_OPERANDS_OUTPUT_IDX (x);

	  if (ASM_OPERANDS_INPUT_LENGTH (x))
	    {
	      for (i = 1; i < ASM_OPERANDS_INPUT_LENGTH (x); i++)
		{
		  hash += (hash_rtx_cb (ASM_OPERANDS_INPUT (x, i),
                                        GET_MODE (ASM_OPERANDS_INPUT (x, i)),
                                        do_not_record_p, hash_arg_in_memory_p,
                                        have_reg_qty, cb)
			   + hash_rtx_string
                           (ASM_OPERANDS_INPUT_CONSTRAINT (x, i)));
		}

	      hash += hash_rtx_string (ASM_OPERANDS_INPUT_CONSTRAINT (x, 0));
	      x = ASM_OPERANDS_INPUT (x, 0);
	      mode = GET_MODE (x);
	      goto repeat;
	    }

	  return hash;
	}
      break;

    default:
      break;
    }

  i = GET_RTX_LENGTH (code) - 1;
  hash += (unsigned) code + (unsigned) GET_MODE (x);
  fmt = GET_RTX_FORMAT (code);
  for (; i >= 0; i--)
    {
      switch (fmt[i])
	{
	case 'e':
	  /* If we are about to do the last recursive call
	     needed at this level, change it into iteration.
	     This function  is called enough to be worth it.  */
	  if (i == 0)
	    {
	      x = XEXP (x, i);
	      goto repeat;
	    }

	  hash += hash_rtx_cb (XEXP (x, i), VOIDmode, do_not_record_p,
                               hash_arg_in_memory_p,
                               have_reg_qty, cb);
	  break;

	case 'E':
	  for (j = 0; j < XVECLEN (x, i); j++)
	    hash += hash_rtx_cb (XVECEXP (x, i, j), VOIDmode, do_not_record_p,
                                 hash_arg_in_memory_p,
                                 have_reg_qty, cb);
	  break;

	case 's':
	  hash += hash_rtx_string (XSTR (x, i));
	  break;

	case 'i':
	  hash += (unsigned int) XINT (x, i);
	  break;

	case 'p':
	  hash += constant_lower_bound (SUBREG_BYTE (x));
	  break;

	case '0': case 't':
	  /* Unused.  */
	  break;

	default:
	  gcc_unreachable ();
	}
    }

  return hash;
}

/* Hash an rtx.  We are careful to make sure the value is never negative.
   Equivalent registers hash identically.
   MODE is used in hashing for CONST_INTs only;
   otherwise the mode of X is used.

   Store 1 in DO_NOT_RECORD_P if any subexpression is volatile.

   If HASH_ARG_IN_MEMORY_P is not NULL, store 1 in it if X contains
   a MEM rtx which does not have the MEM_READONLY_P flag set.

   Note that cse_insn knows that the hash code of a MEM expression
   is just (int) MEM plus the hash code of the address.  */

unsigned
hash_rtx (const_rtx x, machine_mode mode, int *do_not_record_p,
	  int *hash_arg_in_memory_p, bool have_reg_qty)
{
  return hash_rtx_cb (x, mode, do_not_record_p,
                      hash_arg_in_memory_p, have_reg_qty, NULL);
}

/* Hash an rtx X for cse via hash_rtx.
   Stores 1 in do_not_record if any subexpression is volatile.
   Stores 1 in hash_arg_in_memory if X contains a mem rtx which
   does not have the MEM_READONLY_P flag set.  */

static inline unsigned
canon_hash (rtx x, machine_mode mode)
{
  return hash_rtx (x, mode, &do_not_record, &hash_arg_in_memory, true);
}

/* Like canon_hash but with no side effects, i.e. do_not_record
   and hash_arg_in_memory are not changed.  */

static inline unsigned
safe_hash (rtx x, machine_mode mode)
{
  int dummy_do_not_record;
  return hash_rtx (x, mode, &dummy_do_not_record, NULL, true);
}

/* Return 1 iff X and Y would canonicalize into the same thing,
   without actually constructing the canonicalization of either one.
   If VALIDATE is nonzero,
   we assume X is an expression being processed from the rtl
   and Y was found in the hash table.  We check register refs
   in Y for being marked as valid.

   If FOR_GCSE is true, we compare X and Y for equivalence for GCSE.  */

int
exp_equiv_p (const_rtx x, const_rtx y, int validate, bool for_gcse)
{
  int i, j;
  enum rtx_code code;
  const char *fmt;

  /* Note: it is incorrect to assume an expression is equivalent to itself
     if VALIDATE is nonzero.  */
  if (x == y && !validate)
    return 1;

  if (x == 0 || y == 0)
    return x == y;

  code = GET_CODE (x);
  if (code != GET_CODE (y))
    return 0;

  /* (MULT:SI x y) and (MULT:HI x y) are NOT equivalent.  */
  if (GET_MODE (x) != GET_MODE (y))
    return 0;

  /* MEMs referring to different address space are not equivalent.  */
  if (code == MEM && MEM_ADDR_SPACE (x) != MEM_ADDR_SPACE (y))
    return 0;

  switch (code)
    {
    case PC:
    CASE_CONST_UNIQUE:
      return x == y;

    case CONST_VECTOR:
      if (!same_vector_encodings_p (x, y))
	return false;
      break;

    case LABEL_REF:
      return label_ref_label (x) == label_ref_label (y);

    case SYMBOL_REF:
      return XSTR (x, 0) == XSTR (y, 0);

    case REG:
      if (for_gcse)
	return REGNO (x) == REGNO (y);
      else
	{
	  unsigned int regno = REGNO (y);
	  unsigned int i;
	  unsigned int endregno = END_REGNO (y);

	  /* If the quantities are not the same, the expressions are not
	     equivalent.  If there are and we are not to validate, they
	     are equivalent.  Otherwise, ensure all regs are up-to-date.  */

	  if (REG_QTY (REGNO (x)) != REG_QTY (regno))
	    return 0;

	  if (! validate)
	    return 1;

	  for (i = regno; i < endregno; i++)
	    if (REG_IN_TABLE (i) != REG_TICK (i))
	      return 0;

	  return 1;
	}

    case MEM:
      if (for_gcse)
	{
	  /* A volatile mem should not be considered equivalent to any
	     other.  */
	  if (MEM_VOLATILE_P (x) || MEM_VOLATILE_P (y))
	    return 0;

	  /* Can't merge two expressions in different alias sets, since we
	     can decide that the expression is transparent in a block when
	     it isn't, due to it being set with the different alias set.

	     Also, can't merge two expressions with different MEM_ATTRS.
	     They could e.g. be two different entities allocated into the
	     same space on the stack (see e.g. PR25130).  In that case, the
	     MEM addresses can be the same, even though the two MEMs are
	     absolutely not equivalent.

	     But because really all MEM attributes should be the same for
	     equivalent MEMs, we just use the invariant that MEMs that have
	     the same attributes share the same mem_attrs data structure.  */
	  if (!mem_attrs_eq_p (MEM_ATTRS (x), MEM_ATTRS (y)))
	    return 0;

	  /* If we are handling exceptions, we cannot consider two expressions
	     with different trapping status as equivalent, because simple_mem
	     might accept one and reject the other.  */
	  if (cfun->can_throw_non_call_exceptions
	      && (MEM_NOTRAP_P (x) != MEM_NOTRAP_P (y)))
	    return 0;
	}
      break;

    /*  For commutative operations, check both orders.  */
    case PLUS:
    case MULT:
    case AND:
    case IOR:
    case XOR:
    case NE:
    case EQ:
      return ((exp_equiv_p (XEXP (x, 0), XEXP (y, 0),
			     validate, for_gcse)
	       && exp_equiv_p (XEXP (x, 1), XEXP (y, 1),
				validate, for_gcse))
	      || (exp_equiv_p (XEXP (x, 0), XEXP (y, 1),
				validate, for_gcse)
		  && exp_equiv_p (XEXP (x, 1), XEXP (y, 0),
				   validate, for_gcse)));

    case ASM_OPERANDS:
      /* We don't use the generic code below because we want to
	 disregard filename and line numbers.  */

      /* A volatile asm isn't equivalent to any other.  */
      if (MEM_VOLATILE_P (x) || MEM_VOLATILE_P (y))
	return 0;

      if (GET_MODE (x) != GET_MODE (y)
	  || strcmp (ASM_OPERANDS_TEMPLATE (x), ASM_OPERANDS_TEMPLATE (y))
	  || strcmp (ASM_OPERANDS_OUTPUT_CONSTRAINT (x),
		     ASM_OPERANDS_OUTPUT_CONSTRAINT (y))
	  || ASM_OPERANDS_OUTPUT_IDX (x) != ASM_OPERANDS_OUTPUT_IDX (y)
	  || ASM_OPERANDS_INPUT_LENGTH (x) != ASM_OPERANDS_INPUT_LENGTH (y))
	return 0;

      if (ASM_OPERANDS_INPUT_LENGTH (x))
	{
	  for (i = ASM_OPERANDS_INPUT_LENGTH (x) - 1; i >= 0; i--)
	    if (! exp_equiv_p (ASM_OPERANDS_INPUT (x, i),
			       ASM_OPERANDS_INPUT (y, i),
			       validate, for_gcse)
		|| strcmp (ASM_OPERANDS_INPUT_CONSTRAINT (x, i),
			   ASM_OPERANDS_INPUT_CONSTRAINT (y, i)))
	      return 0;
	}

      return 1;

    default:
      break;
    }

  /* Compare the elements.  If any pair of corresponding elements
     fail to match, return 0 for the whole thing.  */

  fmt = GET_RTX_FORMAT (code);
  for (i = GET_RTX_LENGTH (code) - 1; i >= 0; i--)
    {
      switch (fmt[i])
	{
	case 'e':
	  if (! exp_equiv_p (XEXP (x, i), XEXP (y, i),
			      validate, for_gcse))
	    return 0;
	  break;

	case 'E':
	  if (XVECLEN (x, i) != XVECLEN (y, i))
	    return 0;
	  for (j = 0; j < XVECLEN (x, i); j++)
	    if (! exp_equiv_p (XVECEXP (x, i, j), XVECEXP (y, i, j),
				validate, for_gcse))
	      return 0;
	  break;

	case 's':
	  if (strcmp (XSTR (x, i), XSTR (y, i)))
	    return 0;
	  break;

	case 'i':
	  if (XINT (x, i) != XINT (y, i))
	    return 0;
	  break;

	case 'w':
	  if (XWINT (x, i) != XWINT (y, i))
	    return 0;
	  break;

	case 'p':
	  if (maybe_ne (SUBREG_BYTE (x), SUBREG_BYTE (y)))
	    return 0;
	  break;

	case '0':
	case 't':
	  break;

	default:
	  gcc_unreachable ();
	}
    }

  return 1;
}

/* Subroutine of canon_reg.  Pass *XLOC through canon_reg, and validate
   the result if necessary.  INSN is as for canon_reg.  */

static void
validate_canon_reg (rtx *xloc, rtx_insn *insn)
{
  if (*xloc)
    {
      rtx new_rtx = canon_reg (*xloc, insn);

      /* If replacing pseudo with hard reg or vice versa, ensure the
         insn remains valid.  Likewise if the insn has MATCH_DUPs.  */
      gcc_assert (insn && new_rtx);
      validate_change (insn, xloc, new_rtx, 1);
    }
}

/* Canonicalize an expression:
   replace each register reference inside it
   with the "oldest" equivalent register.

   If INSN is nonzero validate_change is used to ensure that INSN remains valid
   after we make our substitution.  The calls are made with IN_GROUP nonzero
   so apply_change_group must be called upon the outermost return from this
   function (unless INSN is zero).  The result of apply_change_group can
   generally be discarded since the changes we are making are optional.  */

static rtx
canon_reg (rtx x, rtx_insn *insn)
{
  int i;
  enum rtx_code code;
  const char *fmt;

  if (x == 0)
    return x;

  code = GET_CODE (x);
  switch (code)
    {
    case PC:
    case CONST:
    CASE_CONST_ANY:
    case SYMBOL_REF:
    case LABEL_REF:
    case ADDR_VEC:
    case ADDR_DIFF_VEC:
      return x;

    case REG:
      {
	int first;
	int q;
	struct qty_table_elem *ent;

	/* Never replace a hard reg, because hard regs can appear
	   in more than one machine mode, and we must preserve the mode
	   of each occurrence.  Also, some hard regs appear in
	   MEMs that are shared and mustn't be altered.  Don't try to
	   replace any reg that maps to a reg of class NO_REGS.  */
	if (REGNO (x) < FIRST_PSEUDO_REGISTER
	    || ! REGNO_QTY_VALID_P (REGNO (x)))
	  return x;

	q = REG_QTY (REGNO (x));
	ent = &qty_table[q];
	first = ent->first_reg;
	return (first >= FIRST_PSEUDO_REGISTER ? regno_reg_rtx[first]
		: REGNO_REG_CLASS (first) == NO_REGS ? x
		: gen_rtx_REG (ent->mode, first));
      }

    default:
      break;
    }

  fmt = GET_RTX_FORMAT (code);
  for (i = GET_RTX_LENGTH (code) - 1; i >= 0; i--)
    {
      int j;

      if (fmt[i] == 'e')
	validate_canon_reg (&XEXP (x, i), insn);
      else if (fmt[i] == 'E')
	for (j = 0; j < XVECLEN (x, i); j++)
	  validate_canon_reg (&XVECEXP (x, i, j), insn);
    }

  return x;
}

/* Given an operation (CODE, *PARG1, *PARG2), where code is a comparison
   operation (EQ, NE, GT, etc.), follow it back through the hash table and
   what values are being compared.

   *PARG1 and *PARG2 are updated to contain the rtx representing the values
   actually being compared.  For example, if *PARG1 was (reg:CC CC_REG) and
   *PARG2 was (const_int 0), *PARG1 and *PARG2 will be set to the objects that
   were compared to produce (reg:CC CC_REG).

   The return value is the comparison operator and is either the code of
   A or the code corresponding to the inverse of the comparison.  */

static enum rtx_code
find_comparison_args (enum rtx_code code, rtx *parg1, rtx *parg2,
		      machine_mode *pmode1, machine_mode *pmode2)
{
  rtx arg1, arg2;
  hash_set<rtx> *visited = NULL;
  /* Set nonzero when we find something of interest.  */
  rtx x = NULL;

  arg1 = *parg1, arg2 = *parg2;

  /* If ARG2 is const0_rtx, see what ARG1 is equivalent to.  */

  while (arg2 == CONST0_RTX (GET_MODE (arg1)))
    {
      int reverse_code = 0;
      struct table_elt *p = 0;

      /* Remember state from previous iteration.  */
      if (x)
	{
	  if (!visited)
	    visited = new hash_set<rtx>;
	  visited->add (x);
	  x = 0;
	}

      /* If arg1 is a COMPARE, extract the comparison arguments from it.  */

      if (GET_CODE (arg1) == COMPARE && arg2 == const0_rtx)
	x = arg1;

      /* If ARG1 is a comparison operator and CODE is testing for
	 STORE_FLAG_VALUE, get the inner arguments.  */

      else if (COMPARISON_P (arg1))
	{
#ifdef FLOAT_STORE_FLAG_VALUE
	  REAL_VALUE_TYPE fsfv;
#endif

	  if (code == NE
	      || (GET_MODE_CLASS (GET_MODE (arg1)) == MODE_INT
		  && code == LT && STORE_FLAG_VALUE == -1)
#ifdef FLOAT_STORE_FLAG_VALUE
	      || (SCALAR_FLOAT_MODE_P (GET_MODE (arg1))
		  && (fsfv = FLOAT_STORE_FLAG_VALUE (GET_MODE (arg1)),
		      REAL_VALUE_NEGATIVE (fsfv)))
#endif
	      )
	    x = arg1;
	  else if (code == EQ
		   || (GET_MODE_CLASS (GET_MODE (arg1)) == MODE_INT
		       && code == GE && STORE_FLAG_VALUE == -1)
#ifdef FLOAT_STORE_FLAG_VALUE
		   || (SCALAR_FLOAT_MODE_P (GET_MODE (arg1))
		       && (fsfv = FLOAT_STORE_FLAG_VALUE (GET_MODE (arg1)),
			   REAL_VALUE_NEGATIVE (fsfv)))
#endif
		   )
	    x = arg1, reverse_code = 1;
	}

      /* ??? We could also check for

	 (ne (and (eq (...) (const_int 1))) (const_int 0))

	 and related forms, but let's wait until we see them occurring.  */

      if (x == 0)
	/* Look up ARG1 in the hash table and see if it has an equivalence
	   that lets us see what is being compared.  */
	p = lookup (arg1, SAFE_HASH (arg1, GET_MODE (arg1)), GET_MODE (arg1));
      if (p)
	{
	  p = p->first_same_value;

	  /* If what we compare is already known to be constant, that is as
	     good as it gets.
	     We need to break the loop in this case, because otherwise we
	     can have an infinite loop when looking at a reg that is known
	     to be a constant which is the same as a comparison of a reg
	     against zero which appears later in the insn stream, which in
	     turn is constant and the same as the comparison of the first reg
	     against zero...  */
	  if (p->is_const)
	    break;
	}

      for (; p; p = p->next_same_value)
	{
	  machine_mode inner_mode = GET_MODE (p->exp);
#ifdef FLOAT_STORE_FLAG_VALUE
	  REAL_VALUE_TYPE fsfv;
#endif

	  /* If the entry isn't valid, skip it.  */
	  if (! exp_equiv_p (p->exp, p->exp, 1, false))
	    continue;

	  /* If it's a comparison we've used before, skip it.  */
	  if (visited && visited->contains (p->exp))
	    continue;

	  if (GET_CODE (p->exp) == COMPARE
	      /* Another possibility is that this machine has a compare insn
		 that includes the comparison code.  In that case, ARG1 would
		 be equivalent to a comparison operation that would set ARG1 to
		 either STORE_FLAG_VALUE or zero.  If this is an NE operation,
		 ORIG_CODE is the actual comparison being done; if it is an EQ,
		 we must reverse ORIG_CODE.  On machine with a negative value
		 for STORE_FLAG_VALUE, also look at LT and GE operations.  */
	      || ((code == NE
		   || (code == LT
		       && val_signbit_known_set_p (inner_mode,
						   STORE_FLAG_VALUE))
#ifdef FLOAT_STORE_FLAG_VALUE
		   || (code == LT
		       && SCALAR_FLOAT_MODE_P (inner_mode)
		       && (fsfv = FLOAT_STORE_FLAG_VALUE (GET_MODE (arg1)),
			   REAL_VALUE_NEGATIVE (fsfv)))
#endif
		   )
		  && COMPARISON_P (p->exp)))
	    {
	      x = p->exp;
	      break;
	    }
	  else if ((code == EQ
		    || (code == GE
			&& val_signbit_known_set_p (inner_mode,
						    STORE_FLAG_VALUE))
#ifdef FLOAT_STORE_FLAG_VALUE
		    || (code == GE
			&& SCALAR_FLOAT_MODE_P (inner_mode)
			&& (fsfv = FLOAT_STORE_FLAG_VALUE (GET_MODE (arg1)),
			    REAL_VALUE_NEGATIVE (fsfv)))
#endif
		    )
		   && COMPARISON_P (p->exp))
	    {
	      reverse_code = 1;
	      x = p->exp;
	      break;
	    }

	  /* If this non-trapping address, e.g. fp + constant, the
	     equivalent is a better operand since it may let us predict
	     the value of the comparison.  */
	  else if (!rtx_addr_can_trap_p (p->exp))
	    {
	      arg1 = p->exp;
	      continue;
	    }
	}

      /* If we didn't find a useful equivalence for ARG1, we are done.
	 Otherwise, set up for the next iteration.  */
      if (x == 0)
	break;

      /* If we need to reverse the comparison, make sure that is
	 possible -- we can't necessarily infer the value of GE from LT
	 with floating-point operands.  */
      if (reverse_code)
	{
	  enum rtx_code reversed = reversed_comparison_code (x, NULL);
	  if (reversed == UNKNOWN)
	    break;
	  else
	    code = reversed;
	}
      else if (COMPARISON_P (x))
	code = GET_CODE (x);
      arg1 = XEXP (x, 0), arg2 = XEXP (x, 1);
    }

  /* Return our results.  Return the modes from before fold_rtx
     because fold_rtx might produce const_int, and then it's too late.  */
  *pmode1 = GET_MODE (arg1), *pmode2 = GET_MODE (arg2);
  *parg1 = fold_rtx (arg1, 0), *parg2 = fold_rtx (arg2, 0);

  if (visited)
    delete visited;
  return code;
}

/* If X is a nontrivial arithmetic operation on an argument for which
   a constant value can be determined, return the result of operating
   on that value, as a constant.  Otherwise, return X, possibly with
   one or more operands changed to a forward-propagated constant.

   If X is a register whose contents are known, we do NOT return
   those contents here; equiv_constant is called to perform that task.
   For SUBREGs and MEMs, we do that both here and in equiv_constant.

   INSN is the insn that we may be modifying.  If it is 0, make a copy
   of X before modifying it.  */

static rtx
fold_rtx (rtx x, rtx_insn *insn)
{
  enum rtx_code code;
  machine_mode mode;
  const char *fmt;
  int i;
  rtx new_rtx = 0;
  int changed = 0;
  poly_int64 xval;

  /* Operands of X.  */
  /* Workaround -Wmaybe-uninitialized false positive during
     profiledbootstrap by initializing them.  */
  rtx folded_arg0 = NULL_RTX;
  rtx folded_arg1 = NULL_RTX;

  /* Constant equivalents of first three operands of X;
     0 when no such equivalent is known.  */
  rtx const_arg0;
  rtx const_arg1;
  rtx const_arg2;

  /* The mode of the first operand of X.  We need this for sign and zero
     extends.  */
  machine_mode mode_arg0;

  if (x == 0)
    return x;

  /* Try to perform some initial simplifications on X.  */
  code = GET_CODE (x);
  switch (code)
    {
    case MEM:
    case SUBREG:
    /* The first operand of a SIGN/ZERO_EXTRACT has a different meaning
       than it would in other contexts.  Basically its mode does not
       signify the size of the object read.  That information is carried
       by size operand.    If we happen to have a MEM of the appropriate
       mode in our tables with a constant value we could simplify the
       extraction incorrectly if we allowed substitution of that value
       for the MEM.   */
    case ZERO_EXTRACT:
    case SIGN_EXTRACT:
      if ((new_rtx = equiv_constant (x)) != NULL_RTX)
        return new_rtx;
      return x;

    case CONST:
    CASE_CONST_ANY:
    case SYMBOL_REF:
    case LABEL_REF:
    case REG:
    case PC:
      /* No use simplifying an EXPR_LIST
	 since they are used only for lists of args
	 in a function call's REG_EQUAL note.  */
    case EXPR_LIST:
      return x;

    case ASM_OPERANDS:
      if (insn)
	{
	  for (i = ASM_OPERANDS_INPUT_LENGTH (x) - 1; i >= 0; i--)
	    validate_change (insn, &ASM_OPERANDS_INPUT (x, i),
			     fold_rtx (ASM_OPERANDS_INPUT (x, i), insn), 0);
	}
      return x;

    case CALL:
      if (NO_FUNCTION_CSE && CONSTANT_P (XEXP (XEXP (x, 0), 0)))
	return x;
      break;
    case VEC_SELECT:
      {
	rtx trueop0 = XEXP (x, 0);
	mode = GET_MODE (trueop0);
	rtx trueop1 = XEXP (x, 1);
	/* If we select a low-part subreg, return that.  */
	if (vec_series_lowpart_p (GET_MODE (x), mode, trueop1))
	  {
	    rtx new_rtx = lowpart_subreg (GET_MODE (x), trueop0, mode);
	    if (new_rtx != NULL_RTX)
	      return new_rtx;
	  }
      }

    /* Anything else goes through the loop below.  */
    default:
      break;
    }

  mode = GET_MODE (x);
  const_arg0 = 0;
  const_arg1 = 0;
  const_arg2 = 0;
  mode_arg0 = VOIDmode;

  /* Try folding our operands.
     Then see which ones have constant values known.  */

  fmt = GET_RTX_FORMAT (code);
  for (i = GET_RTX_LENGTH (code) - 1; i >= 0; i--)
    if (fmt[i] == 'e')
      {
	rtx folded_arg = XEXP (x, i), const_arg;
	machine_mode mode_arg = GET_MODE (folded_arg);

	switch (GET_CODE (folded_arg))
	  {
	  case MEM:
	  case REG:
	  case SUBREG:
	    const_arg = equiv_constant (folded_arg);
	    break;

	  case CONST:
	  CASE_CONST_ANY:
	  case SYMBOL_REF:
	  case LABEL_REF:
	    const_arg = folded_arg;
	    break;

	  default:
	    folded_arg = fold_rtx (folded_arg, insn);
	    const_arg = equiv_constant (folded_arg);
	    break;
	  }

	/* For the first three operands, see if the operand
	   is constant or equivalent to a constant.  */
	switch (i)
	  {
	  case 0:
	    folded_arg0 = folded_arg;
	    const_arg0 = const_arg;
	    mode_arg0 = mode_arg;
	    break;
	  case 1:
	    folded_arg1 = folded_arg;
	    const_arg1 = const_arg;
	    break;
	  case 2:
	    const_arg2 = const_arg;
	    break;
	  }

	/* Pick the least expensive of the argument and an equivalent constant
	   argument.  */
	if (const_arg != 0
	    && const_arg != folded_arg
	    && (COST_IN (const_arg, mode_arg, code, i)
		<= COST_IN (folded_arg, mode_arg, code, i))

	    /* It's not safe to substitute the operand of a conversion
	       operator with a constant, as the conversion's identity
	       depends upon the mode of its operand.  This optimization
	       is handled by the call to simplify_unary_operation.  */
	    && (GET_RTX_CLASS (code) != RTX_UNARY
		|| GET_MODE (const_arg) == mode_arg0
		|| (code != ZERO_EXTEND
		    && code != SIGN_EXTEND
		    && code != TRUNCATE
		    && code != FLOAT_TRUNCATE
		    && code != FLOAT_EXTEND
		    && code != FLOAT
		    && code != FIX
		    && code != UNSIGNED_FLOAT
		    && code != UNSIGNED_FIX)))
	  folded_arg = const_arg;

	if (folded_arg == XEXP (x, i))
	  continue;

	if (insn == NULL_RTX && !changed)
	  x = copy_rtx (x);
	changed = 1;
	validate_unshare_change (insn, &XEXP (x, i), folded_arg, 1);
      }

  if (changed)
    {
      /* Canonicalize X if necessary, and keep const_argN and folded_argN
	 consistent with the order in X.  */
      if (canonicalize_change_group (insn, x))
	{
	  std::swap (const_arg0, const_arg1);
	  std::swap (folded_arg0, folded_arg1);
	}

      apply_change_group ();
    }

  /* If X is an arithmetic operation, see if we can simplify it.  */

  switch (GET_RTX_CLASS (code))
    {
    case RTX_UNARY:
      {
	/* We can't simplify extension ops unless we know the
	   original mode.  */
	if ((code == ZERO_EXTEND || code == SIGN_EXTEND)
	    && mode_arg0 == VOIDmode)
	  break;

	new_rtx = simplify_unary_operation (code, mode,
					    const_arg0 ? const_arg0 : folded_arg0,
					    mode_arg0);
      }
      break;

    case RTX_COMPARE:
    case RTX_COMM_COMPARE:
      /* See what items are actually being compared and set FOLDED_ARG[01]
	 to those values and CODE to the actual comparison code.  If any are
	 constant, set CONST_ARG0 and CONST_ARG1 appropriately.  We needn't
	 do anything if both operands are already known to be constant.  */

      /* ??? Vector mode comparisons are not supported yet.  */
      if (VECTOR_MODE_P (mode))
	break;

      if (const_arg0 == 0 || const_arg1 == 0)
	{
	  struct table_elt *p0, *p1;
	  rtx true_rtx, false_rtx;
	  machine_mode mode_arg1;

	  if (SCALAR_FLOAT_MODE_P (mode))
	    {
#ifdef FLOAT_STORE_FLAG_VALUE
	      true_rtx = (const_double_from_real_value
			  (FLOAT_STORE_FLAG_VALUE (mode), mode));
#else
	      true_rtx = NULL_RTX;
#endif
	      false_rtx = CONST0_RTX (mode);
	    }
	  else
	    {
	      true_rtx = const_true_rtx;
	      false_rtx = const0_rtx;
	    }

	  code = find_comparison_args (code, &folded_arg0, &folded_arg1,
				       &mode_arg0, &mode_arg1);

	  /* If the mode is VOIDmode or a MODE_CC mode, we don't know
	     what kinds of things are being compared, so we can't do
	     anything with this comparison.  */

	  if (mode_arg0 == VOIDmode || GET_MODE_CLASS (mode_arg0) == MODE_CC)
	    break;

	  const_arg0 = equiv_constant (folded_arg0);
	  const_arg1 = equiv_constant (folded_arg1);

	  /* If we do not now have two constants being compared, see
	     if we can nevertheless deduce some things about the
	     comparison.  */
	  if (const_arg0 == 0 || const_arg1 == 0)
	    {
	      if (const_arg1 != NULL)
		{
		  rtx cheapest_simplification;
		  int cheapest_cost;
		  rtx simp_result;
		  struct table_elt *p;

		  /* See if we can find an equivalent of folded_arg0
		     that gets us a cheaper expression, possibly a
		     constant through simplifications.  */
		  p = lookup (folded_arg0, SAFE_HASH (folded_arg0, mode_arg0),
			      mode_arg0);

		  if (p != NULL)
		    {
		      cheapest_simplification = x;
		      cheapest_cost = COST (x, mode);

		      for (p = p->first_same_value; p != NULL; p = p->next_same_value)
			{
			  int cost;

			  /* If the entry isn't valid, skip it.  */
			  if (! exp_equiv_p (p->exp, p->exp, 1, false))
			    continue;

			  /* Try to simplify using this equivalence.  */
			  simp_result
			    = simplify_relational_operation (code, mode,
							     mode_arg0,
							     p->exp,
							     const_arg1);

			  if (simp_result == NULL)
			    continue;

			  cost = COST (simp_result, mode);
			  if (cost < cheapest_cost)
			    {
			      cheapest_cost = cost;
			      cheapest_simplification = simp_result;
			    }
			}

		      /* If we have a cheaper expression now, use that
			 and try folding it further, from the top.  */
		      if (cheapest_simplification != x)
			return fold_rtx (copy_rtx (cheapest_simplification),
					 insn);
		    }
		}

	      /* See if the two operands are the same.  */

	      if ((REG_P (folded_arg0)
		   && REG_P (folded_arg1)
		   && (REG_QTY (REGNO (folded_arg0))
		       == REG_QTY (REGNO (folded_arg1))))
		  || ((p0 = lookup (folded_arg0,
				    SAFE_HASH (folded_arg0, mode_arg0),
				    mode_arg0))
		      && (p1 = lookup (folded_arg1,
				       SAFE_HASH (folded_arg1, mode_arg0),
				       mode_arg0))
		      && p0->first_same_value == p1->first_same_value))
		folded_arg1 = folded_arg0;

	      /* If FOLDED_ARG0 is a register, see if the comparison we are
		 doing now is either the same as we did before or the reverse
		 (we only check the reverse if not floating-point).  */
	      else if (REG_P (folded_arg0))
		{
		  int qty = REG_QTY (REGNO (folded_arg0));

		  if (REGNO_QTY_VALID_P (REGNO (folded_arg0)))
		    {
		      struct qty_table_elem *ent = &qty_table[qty];

		      if ((comparison_dominates_p (ent->comparison_code, code)
			   || (! FLOAT_MODE_P (mode_arg0)
			       && comparison_dominates_p (ent->comparison_code,
						          reverse_condition (code))))
			  && (rtx_equal_p (ent->comparison_const, folded_arg1)
			      || (const_arg1
				  && rtx_equal_p (ent->comparison_const,
						  const_arg1))
			      || (REG_P (folded_arg1)
				  && (REG_QTY (REGNO (folded_arg1)) == ent->comparison_qty))))
			{
			  if (comparison_dominates_p (ent->comparison_code, code))
			    {
			      if (true_rtx)
				return true_rtx;
			      else
				break;
			    }
			  else
			    return false_rtx;
			}
		    }
		}
	    }
	}

      /* If we are comparing against zero, see if the first operand is
	 equivalent to an IOR with a constant.  If so, we may be able to
	 determine the result of this comparison.  */
      if (const_arg1 == const0_rtx && !const_arg0)
	{
	  rtx y = lookup_as_function (folded_arg0, IOR);
	  rtx inner_const;

	  if (y != 0
	      && (inner_const = equiv_constant (XEXP (y, 1))) != 0
	      && CONST_INT_P (inner_const)
	      && INTVAL (inner_const) != 0)
	    folded_arg0 = gen_rtx_IOR (mode_arg0, XEXP (y, 0), inner_const);
	}

      {
	rtx op0 = const_arg0 ? const_arg0 : copy_rtx (folded_arg0);
	rtx op1 = const_arg1 ? const_arg1 : copy_rtx (folded_arg1);
	new_rtx = simplify_relational_operation (code, mode, mode_arg0,
						 op0, op1);
      }
      break;

    case RTX_BIN_ARITH:
    case RTX_COMM_ARITH:
      switch (code)
	{
	case PLUS:
	  /* If the second operand is a LABEL_REF, see if the first is a MINUS
	     with that LABEL_REF as its second operand.  If so, the result is
	     the first operand of that MINUS.  This handles switches with an
	     ADDR_DIFF_VEC table.  */
	  if (const_arg1 && GET_CODE (const_arg1) == LABEL_REF)
	    {
	      rtx y
		= GET_CODE (folded_arg0) == MINUS ? folded_arg0
		: lookup_as_function (folded_arg0, MINUS);

	      if (y != 0 && GET_CODE (XEXP (y, 1)) == LABEL_REF
		  && label_ref_label (XEXP (y, 1)) == label_ref_label (const_arg1))
		return XEXP (y, 0);

	      /* Now try for a CONST of a MINUS like the above.  */
	      if ((y = (GET_CODE (folded_arg0) == CONST ? folded_arg0
			: lookup_as_function (folded_arg0, CONST))) != 0
		  && GET_CODE (XEXP (y, 0)) == MINUS
		  && GET_CODE (XEXP (XEXP (y, 0), 1)) == LABEL_REF
		  && label_ref_label (XEXP (XEXP (y, 0), 1)) == label_ref_label (const_arg1))
		return XEXP (XEXP (y, 0), 0);
	    }

	  /* Likewise if the operands are in the other order.  */
	  if (const_arg0 && GET_CODE (const_arg0) == LABEL_REF)
	    {
	      rtx y
		= GET_CODE (folded_arg1) == MINUS ? folded_arg1
		: lookup_as_function (folded_arg1, MINUS);

	      if (y != 0 && GET_CODE (XEXP (y, 1)) == LABEL_REF
		  && label_ref_label (XEXP (y, 1)) == label_ref_label (const_arg0))
		return XEXP (y, 0);

	      /* Now try for a CONST of a MINUS like the above.  */
	      if ((y = (GET_CODE (folded_arg1) == CONST ? folded_arg1
			: lookup_as_function (folded_arg1, CONST))) != 0
		  && GET_CODE (XEXP (y, 0)) == MINUS
		  && GET_CODE (XEXP (XEXP (y, 0), 1)) == LABEL_REF
		  && label_ref_label (XEXP (XEXP (y, 0), 1)) == label_ref_label (const_arg0))
		return XEXP (XEXP (y, 0), 0);
	    }

	  /* If second operand is a register equivalent to a negative
	     CONST_INT, see if we can find a register equivalent to the
	     positive constant.  Make a MINUS if so.  Don't do this for
	     a non-negative constant since we might then alternate between
	     choosing positive and negative constants.  Having the positive
	     constant previously-used is the more common case.  Be sure
	     the resulting constant is non-negative; if const_arg1 were
	     the smallest negative number this would overflow: depending
	     on the mode, this would either just be the same value (and
	     hence not save anything) or be incorrect.  */
	  if (const_arg1 != 0 && CONST_INT_P (const_arg1)
	      && INTVAL (const_arg1) < 0
	      /* This used to test

	         -INTVAL (const_arg1) >= 0

		 But The Sun V5.0 compilers mis-compiled that test.  So
		 instead we test for the problematic value in a more direct
		 manner and hope the Sun compilers get it correct.  */
	      && INTVAL (const_arg1) !=
	        (HOST_WIDE_INT_1 << (HOST_BITS_PER_WIDE_INT - 1))
	      && REG_P (folded_arg1))
	    {
	      rtx new_const = GEN_INT (-INTVAL (const_arg1));
	      struct table_elt *p
		= lookup (new_const, SAFE_HASH (new_const, mode), mode);

	      if (p)
		for (p = p->first_same_value; p; p = p->next_same_value)
		  if (REG_P (p->exp))
		    return simplify_gen_binary (MINUS, mode, folded_arg0,
						canon_reg (p->exp, NULL));
	    }
	  goto from_plus;

	case MINUS:
	  /* If we have (MINUS Y C), see if Y is known to be (PLUS Z C2).
	     If so, produce (PLUS Z C2-C).  */
	  if (const_arg1 != 0 && poly_int_rtx_p (const_arg1, &xval))
	    {
	      rtx y = lookup_as_function (XEXP (x, 0), PLUS);
	      if (y && poly_int_rtx_p (XEXP (y, 1)))
		return fold_rtx (plus_constant (mode, copy_rtx (y), -xval),
				 NULL);
	    }

	  /* Fall through.  */

	from_plus:
	case SMIN:    case SMAX:      case UMIN:    case UMAX:
	case IOR:     case AND:       case XOR:
	case MULT:
	case ASHIFT:  case LSHIFTRT:  case ASHIFTRT:
	  /* If we have (<op> <reg> <const_int>) for an associative OP and REG
	     is known to be of similar form, we may be able to replace the
	     operation with a combined operation.  This may eliminate the
	     intermediate operation if every use is simplified in this way.
	     Note that the similar optimization done by combine.cc only works
	     if the intermediate operation's result has only one reference.  */

	  if (REG_P (folded_arg0)
	      && const_arg1 && CONST_INT_P (const_arg1))
	    {
	      int is_shift
		= (code == ASHIFT || code == ASHIFTRT || code == LSHIFTRT);
	      rtx y, inner_const, new_const;
	      rtx canon_const_arg1 = const_arg1;
	      enum rtx_code associate_code;

	      if (is_shift
		  && (INTVAL (const_arg1) >= GET_MODE_UNIT_PRECISION (mode)
		      || INTVAL (const_arg1) < 0))
		{
		  if (SHIFT_COUNT_TRUNCATED)
		    canon_const_arg1 = gen_int_shift_amount
		      (mode, (INTVAL (const_arg1)
			      & (GET_MODE_UNIT_BITSIZE (mode) - 1)));
		  else
		    break;
		}

	      y = lookup_as_function (folded_arg0, code);
	      if (y == 0)
		break;

	      /* If we have compiled a statement like
		 "if (x == (x & mask1))", and now are looking at
		 "x & mask2", we will have a case where the first operand
		 of Y is the same as our first operand.  Unless we detect
		 this case, an infinite loop will result.  */
	      if (XEXP (y, 0) == folded_arg0)
		break;

	      inner_const = equiv_constant (fold_rtx (XEXP (y, 1), 0));
	      if (!inner_const || !CONST_INT_P (inner_const))
		break;

	      /* Don't associate these operations if they are a PLUS with the
		 same constant and it is a power of two.  These might be doable
		 with a pre- or post-increment.  Similarly for two subtracts of
		 identical powers of two with post decrement.  */

	      if (code == PLUS && const_arg1 == inner_const
		  && ((HAVE_PRE_INCREMENT
			  && pow2p_hwi (INTVAL (const_arg1)))
		      || (HAVE_POST_INCREMENT
			  && pow2p_hwi (INTVAL (const_arg1)))
		      || (HAVE_PRE_DECREMENT
			  && pow2p_hwi (- INTVAL (const_arg1)))
		      || (HAVE_POST_DECREMENT
			  && pow2p_hwi (- INTVAL (const_arg1)))))
		break;

	      /* ??? Vector mode shifts by scalar
		 shift operand are not supported yet.  */
	      if (is_shift && VECTOR_MODE_P (mode))
                break;

	      if (is_shift
		  && (INTVAL (inner_const) >= GET_MODE_UNIT_PRECISION (mode)
		      || INTVAL (inner_const) < 0))
		{
		  if (SHIFT_COUNT_TRUNCATED)
		    inner_const = gen_int_shift_amount
		      (mode, (INTVAL (inner_const)
			      & (GET_MODE_UNIT_BITSIZE (mode) - 1)));
		  else
		    break;
		}

	      /* Compute the code used to compose the constants.  For example,
		 A-C1-C2 is A-(C1 + C2), so if CODE == MINUS, we want PLUS.  */

	      associate_code = (is_shift || code == MINUS ? PLUS : code);

	      new_const = simplify_binary_operation (associate_code, mode,
						     canon_const_arg1,
						     inner_const);

	      if (new_const == 0)
		break;

	      /* If we are associating shift operations, don't let this
		 produce a shift of the size of the object or larger.
		 This could occur when we follow a sign-extend by a right
		 shift on a machine that does a sign-extend as a pair
		 of shifts.  */

	      if (is_shift
		  && CONST_INT_P (new_const)
		  && INTVAL (new_const) >= GET_MODE_UNIT_PRECISION (mode))
		{
		  /* As an exception, we can turn an ASHIFTRT of this
		     form into a shift of the number of bits - 1.  */
		  if (code == ASHIFTRT)
		    new_const = gen_int_shift_amount
		      (mode, GET_MODE_UNIT_BITSIZE (mode) - 1);
		  else if (!side_effects_p (XEXP (y, 0)))
		    return CONST0_RTX (mode);
		  else
		    break;
		}

	      y = copy_rtx (XEXP (y, 0));

	      /* If Y contains our first operand (the most common way this
		 can happen is if Y is a MEM), we would do into an infinite
		 loop if we tried to fold it.  So don't in that case.  */

	      if (! reg_mentioned_p (folded_arg0, y))
		y = fold_rtx (y, insn);

	      return simplify_gen_binary (code, mode, y, new_const);
	    }
	  break;

	case DIV:       case UDIV:
	  /* ??? The associative optimization performed immediately above is
	     also possible for DIV and UDIV using associate_code of MULT.
	     However, we would need extra code to verify that the
	     multiplication does not overflow, that is, there is no overflow
	     in the calculation of new_const.  */
	  break;

	default:
	  break;
	}

      new_rtx = simplify_binary_operation (code, mode,
				       const_arg0 ? const_arg0 : folded_arg0,
				       const_arg1 ? const_arg1 : folded_arg1);
      break;

    case RTX_OBJ:
      /* (lo_sum (high X) X) is simply X.  */
      if (code == LO_SUM && const_arg0 != 0
	  && GET_CODE (const_arg0) == HIGH
	  && rtx_equal_p (XEXP (const_arg0, 0), const_arg1))
	return const_arg1;
      break;

    case RTX_TERNARY:
    case RTX_BITFIELD_OPS:
      new_rtx = simplify_ternary_operation (code, mode, mode_arg0,
					const_arg0 ? const_arg0 : folded_arg0,
					const_arg1 ? const_arg1 : folded_arg1,
					const_arg2 ? const_arg2 : XEXP (x, 2));
      break;

    default:
      break;
    }

  return new_rtx ? new_rtx : x;
}

/* Return a constant value currently equivalent to X.
   Return 0 if we don't know one.  */

static rtx
equiv_constant (rtx x)
{
  if (REG_P (x)
      && REGNO_QTY_VALID_P (REGNO (x)))
    {
      int x_q = REG_QTY (REGNO (x));
      struct qty_table_elem *x_ent = &qty_table[x_q];

      if (x_ent->const_rtx)
	x = gen_lowpart (GET_MODE (x), x_ent->const_rtx);
    }

  if (x == 0 || CONSTANT_P (x))
    return x;

  if (GET_CODE (x) == SUBREG)
    {
      machine_mode mode = GET_MODE (x);
      machine_mode imode = GET_MODE (SUBREG_REG (x));
      rtx new_rtx;

      /* See if we previously assigned a constant value to this SUBREG.  */
      if ((new_rtx = lookup_as_function (x, CONST_INT)) != 0
	  || (new_rtx = lookup_as_function (x, CONST_WIDE_INT)) != 0
	  || (NUM_POLY_INT_COEFFS > 1
	      && (new_rtx = lookup_as_function (x, CONST_POLY_INT)) != 0)
          || (new_rtx = lookup_as_function (x, CONST_DOUBLE)) != 0
          || (new_rtx = lookup_as_function (x, CONST_FIXED)) != 0)
        return new_rtx;

      /* If we didn't and if doing so makes sense, see if we previously
	 assigned a constant value to the enclosing word mode SUBREG.  */
      if (known_lt (GET_MODE_SIZE (mode), UNITS_PER_WORD)
	  && known_lt (UNITS_PER_WORD, GET_MODE_SIZE (imode)))
	{
	  poly_int64 byte = (SUBREG_BYTE (x)
			     - subreg_lowpart_offset (mode, word_mode));
	  if (known_ge (byte, 0) && multiple_p (byte, UNITS_PER_WORD))
	    {
	      rtx y = gen_rtx_SUBREG (word_mode, SUBREG_REG (x), byte);
	      new_rtx = lookup_as_function (y, CONST_INT);
	      if (new_rtx)
		return gen_lowpart (mode, new_rtx);
	    }
	}

      /* Otherwise see if we already have a constant for the inner REG,
	 and if that is enough to calculate an equivalent constant for
	 the subreg.  Note that the upper bits of paradoxical subregs
	 are undefined, so they cannot be said to equal anything.  */
      if (REG_P (SUBREG_REG (x))
	  && !paradoxical_subreg_p (x)
	  && (new_rtx = equiv_constant (SUBREG_REG (x))) != 0)
        return simplify_subreg (mode, new_rtx, imode, SUBREG_BYTE (x));

      return 0;
    }

  /* If X is a MEM, see if it is a constant-pool reference, or look it up in
     the hash table in case its value was seen before.  */

  if (MEM_P (x))
    {
      struct table_elt *elt;

      x = avoid_constant_pool_reference (x);
      if (CONSTANT_P (x))
	return x;

      elt = lookup (x, SAFE_HASH (x, GET_MODE (x)), GET_MODE (x));
      if (elt == 0)
	return 0;

      for (elt = elt->first_same_value; elt; elt = elt->next_same_value)
	if (elt->is_const && CONSTANT_P (elt->exp))
	  return elt->exp;
    }

  return 0;
}

/* Given INSN, a jump insn, TAKEN indicates if we are following the
   "taken" branch.

   In certain cases, this can cause us to add an equivalence.  For example,
   if we are following the taken case of
	if (i == 2)
   we can add the fact that `i' and '2' are now equivalent.

   In any case, we can record that this comparison was passed.  If the same
   comparison is seen later, we will know its value.  */

static void
record_jump_equiv (rtx_insn *insn, bool taken)
{
  int cond_known_true;
  rtx op0, op1;
  rtx set;
  machine_mode mode, mode0, mode1;
  int reversed_nonequality = 0;
  enum rtx_code code;

  /* Ensure this is the right kind of insn.  */
  gcc_assert (any_condjump_p (insn));

  set = pc_set (insn);

  /* See if this jump condition is known true or false.  */
  if (taken)
    cond_known_true = (XEXP (SET_SRC (set), 2) == pc_rtx);
  else
    cond_known_true = (XEXP (SET_SRC (set), 1) == pc_rtx);

  /* Get the type of comparison being done and the operands being compared.
     If we had to reverse a non-equality condition, record that fact so we
     know that it isn't valid for floating-point.  */
  code = GET_CODE (XEXP (SET_SRC (set), 0));
  op0 = fold_rtx (XEXP (XEXP (SET_SRC (set), 0), 0), insn);
  op1 = fold_rtx (XEXP (XEXP (SET_SRC (set), 0), 1), insn);

  /* If fold_rtx returns NULL_RTX, there's nothing to record.  */
  if (op0 == NULL_RTX || op1 == NULL_RTX)
    return;

  code = find_comparison_args (code, &op0, &op1, &mode0, &mode1);
  if (! cond_known_true)
    {
      code = reversed_comparison_code_parts (code, op0, op1, insn);

      /* Don't remember if we can't find the inverse.  */
      if (code == UNKNOWN)
	return;
    }

  /* The mode is the mode of the non-constant.  */
  mode = mode0;
  if (mode1 != VOIDmode)
    mode = mode1;

  record_jump_cond (code, mode, op0, op1, reversed_nonequality);
}

/* Yet another form of subreg creation.  In this case, we want something in
   MODE, and we should assume OP has MODE iff it is naturally modeless.  */

static rtx
record_jump_cond_subreg (machine_mode mode, rtx op)
{
  machine_mode op_mode = GET_MODE (op);
  if (op_mode == mode || op_mode == VOIDmode)
    return op;
  return lowpart_subreg (mode, op, op_mode);
}

/* We know that comparison CODE applied to OP0 and OP1 in MODE is true.
   REVERSED_NONEQUALITY is nonzero if CODE had to be swapped.
   Make any useful entries we can with that information.  Called from
   above function and called recursively.  */

static void
record_jump_cond (enum rtx_code code, machine_mode mode, rtx op0,
		  rtx op1, int reversed_nonequality)
{
  unsigned op0_hash, op1_hash;
  int op0_in_memory, op1_in_memory;
  struct table_elt *op0_elt, *op1_elt;

  /* If OP0 and OP1 are known equal, and either is a paradoxical SUBREG,
     we know that they are also equal in the smaller mode (this is also
     true for all smaller modes whether or not there is a SUBREG, but
     is not worth testing for with no SUBREG).  */

  /* Note that GET_MODE (op0) may not equal MODE.  */
  if (code == EQ && paradoxical_subreg_p (op0))
    {
      machine_mode inner_mode = GET_MODE (SUBREG_REG (op0));
      rtx tem = record_jump_cond_subreg (inner_mode, op1);
      if (tem)
	record_jump_cond (code, mode, SUBREG_REG (op0), tem,
			  reversed_nonequality);
    }

  if (code == EQ && paradoxical_subreg_p (op1))
    {
      machine_mode inner_mode = GET_MODE (SUBREG_REG (op1));
      rtx tem = record_jump_cond_subreg (inner_mode, op0);
      if (tem)
	record_jump_cond (code, mode, SUBREG_REG (op1), tem,
			  reversed_nonequality);
    }

  /* Similarly, if this is an NE comparison, and either is a SUBREG
     making a smaller mode, we know the whole thing is also NE.  */

  /* Note that GET_MODE (op0) may not equal MODE;
     if we test MODE instead, we can get an infinite recursion
     alternating between two modes each wider than MODE.  */

  if (code == NE
      && partial_subreg_p (op0)
      && subreg_lowpart_p (op0))
    {
      machine_mode inner_mode = GET_MODE (SUBREG_REG (op0));
      rtx tem = record_jump_cond_subreg (inner_mode, op1);
      if (tem)
	record_jump_cond (code, mode, SUBREG_REG (op0), tem,
			  reversed_nonequality);
    }

  if (code == NE
      && partial_subreg_p (op1)
      && subreg_lowpart_p (op1))
    {
      machine_mode inner_mode = GET_MODE (SUBREG_REG (op1));
      rtx tem = record_jump_cond_subreg (inner_mode, op0);
      if (tem)
	record_jump_cond (code, mode, SUBREG_REG (op1), tem,
			  reversed_nonequality);
    }

  /* Hash both operands.  */

  do_not_record = 0;
  hash_arg_in_memory = 0;
  op0_hash = HASH (op0, mode);
  op0_in_memory = hash_arg_in_memory;

  if (do_not_record)
    return;

  do_not_record = 0;
  hash_arg_in_memory = 0;
  op1_hash = HASH (op1, mode);
  op1_in_memory = hash_arg_in_memory;

  if (do_not_record)
    return;

  /* Look up both operands.  */
  op0_elt = lookup (op0, op0_hash, mode);
  op1_elt = lookup (op1, op1_hash, mode);

  /* If both operands are already equivalent or if they are not in the
     table but are identical, do nothing.  */
  if ((op0_elt != 0 && op1_elt != 0
       && op0_elt->first_same_value == op1_elt->first_same_value)
      || op0 == op1 || rtx_equal_p (op0, op1))
    return;

  /* If we aren't setting two things equal all we can do is save this
     comparison.   Similarly if this is floating-point.  In the latter
     case, OP1 might be zero and both -0.0 and 0.0 are equal to it.
     If we record the equality, we might inadvertently delete code
     whose intent was to change -0 to +0.  */

  if (code != EQ || FLOAT_MODE_P (GET_MODE (op0)))
    {
      struct qty_table_elem *ent;
      int qty;

      /* If we reversed a floating-point comparison, if OP0 is not a
	 register, or if OP1 is neither a register or constant, we can't
	 do anything.  */

      if (!REG_P (op1))
	op1 = equiv_constant (op1);

      if ((reversed_nonequality && FLOAT_MODE_P (mode))
	  || !REG_P (op0) || op1 == 0)
	return;

      /* Put OP0 in the hash table if it isn't already.  This gives it a
	 new quantity number.  */
      if (op0_elt == 0)
	{
	  if (insert_regs (op0, NULL, 0))
	    {
	      rehash_using_reg (op0);
	      op0_hash = HASH (op0, mode);

	      /* If OP0 is contained in OP1, this changes its hash code
		 as well.  Faster to rehash than to check, except
		 for the simple case of a constant.  */
	      if (! CONSTANT_P (op1))
		op1_hash = HASH (op1,mode);
	    }

	  op0_elt = insert (op0, NULL, op0_hash, mode);
	  op0_elt->in_memory = op0_in_memory;
	}

      qty = REG_QTY (REGNO (op0));
      ent = &qty_table[qty];

      ent->comparison_code = code;
      if (REG_P (op1))
	{
	  /* Look it up again--in case op0 and op1 are the same.  */
	  op1_elt = lookup (op1, op1_hash, mode);

	  /* Put OP1 in the hash table so it gets a new quantity number.  */
	  if (op1_elt == 0)
	    {
	      if (insert_regs (op1, NULL, 0))
		{
		  rehash_using_reg (op1);
		  op1_hash = HASH (op1, mode);
		}

	      op1_elt = insert (op1, NULL, op1_hash, mode);
	      op1_elt->in_memory = op1_in_memory;
	    }

	  ent->comparison_const = NULL_RTX;
	  ent->comparison_qty = REG_QTY (REGNO (op1));
	}
      else
	{
	  ent->comparison_const = op1;
	  ent->comparison_qty = -1;
	}

      return;
    }

  /* If either side is still missing an equivalence, make it now,
     then merge the equivalences.  */

  if (op0_elt == 0)
    {
      if (insert_regs (op0, NULL, 0))
	{
	  rehash_using_reg (op0);
	  op0_hash = HASH (op0, mode);
	}

      op0_elt = insert (op0, NULL, op0_hash, mode);
      op0_elt->in_memory = op0_in_memory;
    }

  if (op1_elt == 0)
    {
      if (insert_regs (op1, NULL, 0))
	{
	  rehash_using_reg (op1);
	  op1_hash = HASH (op1, mode);
	}

      op1_elt = insert (op1, NULL, op1_hash, mode);
      op1_elt->in_memory = op1_in_memory;
    }

  merge_equiv_classes (op0_elt, op1_elt);
}

/* CSE processing for one instruction.

   Most "true" common subexpressions are mostly optimized away in GIMPLE,
   but the few that "leak through" are cleaned up by cse_insn, and complex
   addressing modes are often formed here.

   The main function is cse_insn, and between here and that function
   a couple of helper functions is defined to keep the size of cse_insn
   within reasonable proportions.

   Data is shared between the main and helper functions via STRUCT SET,
   that contains all data related for every set in the instruction that
   is being processed.

   Note that cse_main processes all sets in the instruction.  Most
   passes in GCC only process simple SET insns or single_set insns, but
   CSE processes insns with multiple sets as well.  */

/* Data on one SET contained in the instruction.  */

struct set
{
  /* The SET rtx itself.  */
  rtx rtl;
  /* The SET_SRC of the rtx (the original value, if it is changing).  */
  rtx src;
  /* The hash-table element for the SET_SRC of the SET.  */
  struct table_elt *src_elt;
  /* Hash value for the SET_SRC.  */
  unsigned src_hash;
  /* Hash value for the SET_DEST.  */
  unsigned dest_hash;
  /* The SET_DEST, with SUBREG, etc., stripped.  */
  rtx inner_dest;
  /* Nonzero if the SET_SRC is in memory.  */
  char src_in_memory;
  /* Nonzero if the SET_SRC contains something
     whose value cannot be predicted and understood.  */
  char src_volatile;
  /* Original machine mode, in case it becomes a CONST_INT.
     The size of this field should match the size of the mode
     field of struct rtx_def (see rtl.h).  */
  ENUM_BITFIELD(machine_mode) mode : 8;
  /* Hash value of constant equivalent for SET_SRC.  */
  unsigned src_const_hash;
  /* A constant equivalent for SET_SRC, if any.  */
  rtx src_const;
  /* Table entry for constant equivalent for SET_SRC, if any.  */
  struct table_elt *src_const_elt;
  /* Table entry for the destination address.  */
  struct table_elt *dest_addr_elt;
};

/* Special handling for (set REG0 REG1) where REG0 is the
   "cheapest", cheaper than REG1.  After cse, REG1 will probably not
   be used in the sequel, so (if easily done) change this insn to
   (set REG1 REG0) and replace REG1 with REG0 in the previous insn
   that computed their value.  Then REG1 will become a dead store
   and won't cloud the situation for later optimizations.

   Do not make this change if REG1 is a hard register, because it will
   then be used in the sequel and we may be changing a two-operand insn
   into a three-operand insn.

   This is the last transformation that cse_insn will try to do.  */

static void
try_back_substitute_reg (rtx set, rtx_insn *insn)
{
  rtx dest = SET_DEST (set);
  rtx src = SET_SRC (set);

  if (REG_P (dest)
      && REG_P (src) && ! HARD_REGISTER_P (src)
      && REGNO_QTY_VALID_P (REGNO (src)))
    {
      int src_q = REG_QTY (REGNO (src));
      struct qty_table_elem *src_ent = &qty_table[src_q];

      if (src_ent->first_reg == REGNO (dest))
	{
	  /* Scan for the previous nonnote insn, but stop at a basic
	     block boundary.  */
	  rtx_insn *prev = insn;
	  rtx_insn *bb_head = BB_HEAD (BLOCK_FOR_INSN (insn));
	  do
	    {
	      prev = PREV_INSN (prev);
	    }
	  while (prev != bb_head && (NOTE_P (prev) || DEBUG_INSN_P (prev)));

	  /* Do not swap the registers around if the previous instruction
	     attaches a REG_EQUIV note to REG1.

	     ??? It's not entirely clear whether we can transfer a REG_EQUIV
	     from the pseudo that originally shadowed an incoming argument
	     to another register.  Some uses of REG_EQUIV might rely on it
	     being attached to REG1 rather than REG2.

	     This section previously turned the REG_EQUIV into a REG_EQUAL
	     note.  We cannot do that because REG_EQUIV may provide an
	     uninitialized stack slot when REG_PARM_STACK_SPACE is used.  */
	  if (NONJUMP_INSN_P (prev)
	      && GET_CODE (PATTERN (prev)) == SET
	      && SET_DEST (PATTERN (prev)) == src
	      && ! find_reg_note (prev, REG_EQUIV, NULL_RTX))
	    {
	      rtx note;

	      validate_change (prev, &SET_DEST (PATTERN (prev)), dest, 1);
	      validate_change (insn, &SET_DEST (set), src, 1);
	      validate_change (insn, &SET_SRC (set), dest, 1);
	      apply_change_group ();

	      /* If INSN has a REG_EQUAL note, and this note mentions
		 REG0, then we must delete it, because the value in
		 REG0 has changed.  If the note's value is REG1, we must
		 also delete it because that is now this insn's dest.  */
	      note = find_reg_note (insn, REG_EQUAL, NULL_RTX);
	      if (note != 0
		  && (reg_mentioned_p (dest, XEXP (note, 0))
		      || rtx_equal_p (src, XEXP (note, 0))))
		remove_note (insn, note);

	      /* If INSN has a REG_ARGS_SIZE note, move it to PREV.  */
	      note = find_reg_note (insn, REG_ARGS_SIZE, NULL_RTX);
	      if (note != 0)
		{
		  remove_note (insn, note);
		  gcc_assert (!find_reg_note (prev, REG_ARGS_SIZE, NULL_RTX));
		  set_unique_reg_note (prev, REG_ARGS_SIZE, XEXP (note, 0));
		}
	    }
	}
    }
}

/* Add an entry containing RTL X into SETS.  */
static inline void
add_to_set (vec<struct set> *sets, rtx x)
{
  struct set entry = {};
  entry.rtl = x;
  sets->safe_push (entry);
}

/* Record all the SETs in this instruction into SETS_PTR,
   and return the number of recorded sets.  */
static int
find_sets_in_insn (rtx_insn *insn, vec<struct set> *psets)
{
  rtx x = PATTERN (insn);

  if (GET_CODE (x) == SET)
    {
      /* Ignore SETs that are unconditional jumps.
	 They never need cse processing, so this does not hurt.
	 The reason is not efficiency but rather
	 so that we can test at the end for instructions
	 that have been simplified to unconditional jumps
	 and not be misled by unchanged instructions
	 that were unconditional jumps to begin with.  */
      if (SET_DEST (x) == pc_rtx
	  && GET_CODE (SET_SRC (x)) == LABEL_REF)
	;
      /* Don't count call-insns, (set (reg 0) (call ...)), as a set.
	 The hard function value register is used only once, to copy to
	 someplace else, so it isn't worth cse'ing.  */
      else if (GET_CODE (SET_SRC (x)) == CALL)
	;
      else if (GET_CODE (SET_SRC (x)) == CONST_VECTOR
	       && GET_MODE_CLASS (GET_MODE (SET_SRC (x))) != MODE_VECTOR_BOOL
	       /* Prevent duplicates from being generated if the type is a V1
		  type and a subreg.  Folding this will result in the same
		  element as folding x itself.  */
	       && !(SUBREG_P (SET_DEST (x))
		    && known_eq (GET_MODE_NUNITS (GET_MODE (SET_SRC (x))), 1)))
	{
	  /* First register the vector itself.  */
	  add_to_set (psets, x);
	  rtx src = SET_SRC (x);
	  /* Go over the constants of the CONST_VECTOR in forward order, to
	     put them in the same order in the SETS array.  */
	  for (unsigned i = 0; i < const_vector_encoded_nelts (src) ; i++)
	    {
	      /* These are templates and don't actually get emitted but are
		 used to tell CSE how to get to a particular constant.  */
	      rtx y = simplify_gen_vec_select (SET_DEST (x), i);
	      gcc_assert (y);
	      add_to_set (psets, gen_rtx_SET (y, CONST_VECTOR_ELT (src, i)));
	    }
	}
      else
	add_to_set (psets, x);
    }
  else if (GET_CODE (x) == PARALLEL)
    {
      int i, lim = XVECLEN (x, 0);

      /* Go over the expressions of the PARALLEL in forward order, to
	 put them in the same order in the SETS array.  */
      for (i = 0; i < lim; i++)
	{
	  rtx y = XVECEXP (x, 0, i);
	  if (GET_CODE (y) == SET)
	    {
	      /* As above, we ignore unconditional jumps and call-insns and
		 ignore the result of apply_change_group.  */
	      if (SET_DEST (y) == pc_rtx
		  && GET_CODE (SET_SRC (y)) == LABEL_REF)
		;
	      else if (GET_CODE (SET_SRC (y)) == CALL)
		;
	      else
		add_to_set (psets, y);
	    }
	}
    }

  return psets->length ();
}

/* Subroutine of canonicalize_insn.  X is an ASM_OPERANDS in INSN.  */

static void
canon_asm_operands (rtx x, rtx_insn *insn)
{
  for (int i = ASM_OPERANDS_INPUT_LENGTH (x) - 1; i >= 0; i--)
    {
      rtx input = ASM_OPERANDS_INPUT (x, i);
      if (!(REG_P (input) && HARD_REGISTER_P (input)))
	{
	  input = canon_reg (input, insn);
	  validate_change (insn, &ASM_OPERANDS_INPUT (x, i), input, 1);
	}
    }
}

/* Where possible, substitute every register reference in the N_SETS
   number of SETS in INSN with the canonical register.

   Register canonicalization propagatest the earliest register (i.e.
   one that is set before INSN) with the same value.  This is a very
   useful, simple form of CSE, to clean up warts from expanding GIMPLE
   to RTL.  For instance, a CONST for an address is usually expanded
   multiple times to loads into different registers, thus creating many
   subexpressions of the form:

   (set (reg1) (some_const))
   (set (mem (... reg1 ...) (thing)))
   (set (reg2) (some_const))
   (set (mem (... reg2 ...) (thing)))

   After canonicalizing, the code takes the following form:

   (set (reg1) (some_const))
   (set (mem (... reg1 ...) (thing)))
   (set (reg2) (some_const))
   (set (mem (... reg1 ...) (thing)))

   The set to reg2 is now trivially dead, and the memory reference (or
   address, or whatever) may be a candidate for further CSEing.

   In this function, the result of apply_change_group can be ignored;
   see canon_reg.  */

static void
canonicalize_insn (rtx_insn *insn, vec<struct set> *psets)
{
  vec<struct set> sets = *psets;
  int n_sets = sets.length ();
  rtx tem;
  rtx x = PATTERN (insn);
  int i;

  if (CALL_P (insn))
    {
      for (tem = CALL_INSN_FUNCTION_USAGE (insn); tem; tem = XEXP (tem, 1))
	if (GET_CODE (XEXP (tem, 0)) != SET)
	  XEXP (tem, 0) = canon_reg (XEXP (tem, 0), insn);
    }

  if (GET_CODE (x) == SET && GET_CODE (SET_SRC (x)) == CALL)
    {
      canon_reg (SET_SRC (x), insn);
      apply_change_group ();
      fold_rtx (SET_SRC (x), insn);
    }
  else if (GET_CODE (x) == CLOBBER)
    {
      /* If we clobber memory, canon the address.
	 This does nothing when a register is clobbered
	 because we have already invalidated the reg.  */
      if (MEM_P (XEXP (x, 0)))
	canon_reg (XEXP (x, 0), insn);
    }
  else if (GET_CODE (x) == USE
	   && ! (REG_P (XEXP (x, 0))
		 && REGNO (XEXP (x, 0)) < FIRST_PSEUDO_REGISTER))
    /* Canonicalize a USE of a pseudo register or memory location.  */
    canon_reg (x, insn);
  else if (GET_CODE (x) == ASM_OPERANDS)
    canon_asm_operands (x, insn);
  else if (GET_CODE (x) == CALL)
    {
      canon_reg (x, insn);
      apply_change_group ();
      fold_rtx (x, insn);
    }
  else if (DEBUG_INSN_P (insn))
    canon_reg (PATTERN (insn), insn);
  else if (GET_CODE (x) == PARALLEL)
    {
      for (i = XVECLEN (x, 0) - 1; i >= 0; i--)
	{
	  rtx y = XVECEXP (x, 0, i);
	  if (GET_CODE (y) == SET && GET_CODE (SET_SRC (y)) == CALL)
	    {
	      canon_reg (SET_SRC (y), insn);
	      apply_change_group ();
	      fold_rtx (SET_SRC (y), insn);
	    }
	  else if (GET_CODE (y) == CLOBBER)
	    {
	      if (MEM_P (XEXP (y, 0)))
		canon_reg (XEXP (y, 0), insn);
	    }
	  else if (GET_CODE (y) == USE
		   && ! (REG_P (XEXP (y, 0))
			 && REGNO (XEXP (y, 0)) < FIRST_PSEUDO_REGISTER))
	    canon_reg (y, insn);
	  else if (GET_CODE (y) == ASM_OPERANDS)
	    canon_asm_operands (y, insn);
	  else if (GET_CODE (y) == CALL)
	    {
	      canon_reg (y, insn);
	      apply_change_group ();
	      fold_rtx (y, insn);
	    }
	}
    }

  if (n_sets == 1 && REG_NOTES (insn) != 0
      && (tem = find_reg_note (insn, REG_EQUAL, NULL_RTX)) != 0)
    {
      /* We potentially will process this insn many times.  Therefore,
	 drop the REG_EQUAL note if it is equal to the SET_SRC of the
	 unique set in INSN.

	 Do not do so if the REG_EQUAL note is for a STRICT_LOW_PART,
	 because cse_insn handles those specially.  */
      if (GET_CODE (SET_DEST (sets[0].rtl)) != STRICT_LOW_PART
	  && rtx_equal_p (XEXP (tem, 0), SET_SRC (sets[0].rtl)))
	remove_note (insn, tem);
      else
	{
	  canon_reg (XEXP (tem, 0), insn);
	  apply_change_group ();
	  XEXP (tem, 0) = fold_rtx (XEXP (tem, 0), insn);
	  df_notes_rescan (insn);
	}
    }

  /* Canonicalize sources and addresses of destinations.
     We do this in a separate pass to avoid problems when a MATCH_DUP is
     present in the insn pattern.  In that case, we want to ensure that
     we don't break the duplicate nature of the pattern.  So we will replace
     both operands at the same time.  Otherwise, we would fail to find an
     equivalent substitution in the loop calling validate_change below.

     We used to suppress canonicalization of DEST if it appears in SRC,
     but we don't do this any more.  */

  for (i = 0; i < n_sets; i++)
    {
      rtx dest = SET_DEST (sets[i].rtl);
      rtx src = SET_SRC (sets[i].rtl);
      rtx new_rtx = canon_reg (src, insn);

      validate_change (insn, &SET_SRC (sets[i].rtl), new_rtx, 1);

      if (GET_CODE (dest) == ZERO_EXTRACT)
	{
	  validate_change (insn, &XEXP (dest, 1),
			   canon_reg (XEXP (dest, 1), insn), 1);
	  validate_change (insn, &XEXP (dest, 2),
			   canon_reg (XEXP (dest, 2), insn), 1);
	}

      while (GET_CODE (dest) == SUBREG
	     || GET_CODE (dest) == ZERO_EXTRACT
	     || GET_CODE (dest) == STRICT_LOW_PART)
	dest = XEXP (dest, 0);

      if (MEM_P (dest))
	canon_reg (dest, insn);
    }

  /* Now that we have done all the replacements, we can apply the change
     group and see if they all work.  Note that this will cause some
     canonicalizations that would have worked individually not to be applied
     because some other canonicalization didn't work, but this should not
     occur often.

     The result of apply_change_group can be ignored; see canon_reg.  */

  apply_change_group ();
}

/* Main function of CSE.
   First simplify sources and addresses of all assignments
   in the instruction, using previously-computed equivalents values.
   Then install the new sources and destinations in the table
   of available values.  */

static void
cse_insn (rtx_insn *insn)
{
  rtx x = PATTERN (insn);
  int i;
  rtx tem;
  int n_sets = 0;

  rtx src_eqv = 0;
  struct table_elt *src_eqv_elt = 0;
  int src_eqv_volatile = 0;
  int src_eqv_in_memory = 0;
  unsigned src_eqv_hash = 0;

  this_insn = insn;

  /* Find all regs explicitly clobbered in this insn,
     to ensure they are not replaced with any other regs
     elsewhere in this insn.  */
  invalidate_from_sets_and_clobbers (insn);

  /* Record all the SETs in this instruction.  */
  auto_vec<struct set, 8> sets;
  n_sets = find_sets_in_insn (insn, (vec<struct set>*)&sets);

  /* Substitute the canonical register where possible.  */
  canonicalize_insn (insn, (vec<struct set>*)&sets);

  /* If this insn has a REG_EQUAL note, store the equivalent value in SRC_EQV,
     if different, or if the DEST is a STRICT_LOW_PART/ZERO_EXTRACT.  The
     latter condition is necessary because SRC_EQV is handled specially for
     this case, and if it isn't set, then there will be no equivalence
     for the destination.  */
  if (n_sets == 1 && REG_NOTES (insn) != 0
      && (tem = find_reg_note (insn, REG_EQUAL, NULL_RTX)) != 0)
    {

      if (GET_CODE (SET_DEST (sets[0].rtl)) != ZERO_EXTRACT
	  && (! rtx_equal_p (XEXP (tem, 0), SET_SRC (sets[0].rtl))
	      || GET_CODE (SET_DEST (sets[0].rtl)) == STRICT_LOW_PART))
	src_eqv = copy_rtx (XEXP (tem, 0));
      /* If DEST is of the form ZERO_EXTACT, as in:
	 (set (zero_extract:SI (reg:SI 119)
		  (const_int 16 [0x10])
		  (const_int 16 [0x10]))
	      (const_int 51154 [0xc7d2]))
	 REG_EQUAL note will specify the value of register (reg:SI 119) at this
	 point.  Note that this is different from SRC_EQV. We can however
	 calculate SRC_EQV with the position and width of ZERO_EXTRACT.  */
      else if (GET_CODE (SET_DEST (sets[0].rtl)) == ZERO_EXTRACT
	       && CONST_INT_P (XEXP (tem, 0))
	       && CONST_INT_P (XEXP (SET_DEST (sets[0].rtl), 1))
	       && CONST_INT_P (XEXP (SET_DEST (sets[0].rtl), 2)))
	{
	  rtx dest_reg = XEXP (SET_DEST (sets[0].rtl), 0);
	  /* This is the mode of XEXP (tem, 0) as well.  */
	  scalar_int_mode dest_mode
	    = as_a <scalar_int_mode> (GET_MODE (dest_reg));
	  rtx width = XEXP (SET_DEST (sets[0].rtl), 1);
	  rtx pos = XEXP (SET_DEST (sets[0].rtl), 2);
	  HOST_WIDE_INT val = INTVAL (XEXP (tem, 0));
	  HOST_WIDE_INT mask;
	  unsigned int shift;
	  if (BITS_BIG_ENDIAN)
	    shift = (GET_MODE_PRECISION (dest_mode)
		     - INTVAL (pos) - INTVAL (width));
	  else
	    shift = INTVAL (pos);
	  if (INTVAL (width) == HOST_BITS_PER_WIDE_INT)
	    mask = HOST_WIDE_INT_M1;
	  else
	    mask = (HOST_WIDE_INT_1 << INTVAL (width)) - 1;
	  val = (val >> shift) & mask;
	  src_eqv = GEN_INT (val);
	}
    }

  /* Set sets[i].src_elt to the class each source belongs to.
     Detect assignments from or to volatile things
     and set set[i] to zero so they will be ignored
     in the rest of this function.

     Nothing in this loop changes the hash table or the register chains.  */

  for (i = 0; i < n_sets; i++)
    {
      bool repeat = false;
      bool noop_insn = false;
      rtx src, dest;
      rtx src_folded;
      struct table_elt *elt = 0, *p;
      machine_mode mode;
      rtx src_eqv_here;
      rtx src_const = 0;
      rtx src_related = 0;
      bool src_related_is_const_anchor = false;
      struct table_elt *src_const_elt = 0;
      int src_cost = MAX_COST;
      int src_eqv_cost = MAX_COST;
      int src_folded_cost = MAX_COST;
      int src_related_cost = MAX_COST;
      int src_elt_cost = MAX_COST;
      int src_regcost = MAX_COST;
      int src_eqv_regcost = MAX_COST;
      int src_folded_regcost = MAX_COST;
      int src_related_regcost = MAX_COST;
      int src_elt_regcost = MAX_COST;
      scalar_int_mode int_mode;

      dest = SET_DEST (sets[i].rtl);
      src = SET_SRC (sets[i].rtl);

      /* If SRC is a constant that has no machine mode,
	 hash it with the destination's machine mode.
	 This way we can keep different modes separate.  */

      mode = GET_MODE (src) == VOIDmode ? GET_MODE (dest) : GET_MODE (src);
      sets[i].mode = mode;

      if (src_eqv)
	{
	  machine_mode eqvmode = mode;
	  if (GET_CODE (dest) == STRICT_LOW_PART)
	    eqvmode = GET_MODE (SUBREG_REG (XEXP (dest, 0)));
	  do_not_record = 0;
	  hash_arg_in_memory = 0;
	  src_eqv_hash = HASH (src_eqv, eqvmode);

	  /* Find the equivalence class for the equivalent expression.  */

	  if (!do_not_record)
	    src_eqv_elt = lookup (src_eqv, src_eqv_hash, eqvmode);

	  src_eqv_volatile = do_not_record;
	  src_eqv_in_memory = hash_arg_in_memory;
	}

      /* If this is a STRICT_LOW_PART assignment, src_eqv corresponds to the
	 value of the INNER register, not the destination.  So it is not
	 a valid substitution for the source.  But save it for later.  */
      if (GET_CODE (dest) == STRICT_LOW_PART)
	src_eqv_here = 0;
      else
	src_eqv_here = src_eqv;

      /* Simplify and foldable subexpressions in SRC.  Then get the fully-
	 simplified result, which may not necessarily be valid.  */
      src_folded = fold_rtx (src, NULL);

#if 0
      /* ??? This caused bad code to be generated for the m68k port with -O2.
	 Suppose src is (CONST_INT -1), and that after truncation src_folded
	 is (CONST_INT 3).  Suppose src_folded is then used for src_const.
	 At the end we will add src and src_const to the same equivalence
	 class.  We now have 3 and -1 on the same equivalence class.  This
	 causes later instructions to be mis-optimized.  */
      /* If storing a constant in a bitfield, pre-truncate the constant
	 so we will be able to record it later.  */
      if (GET_CODE (SET_DEST (sets[i].rtl)) == ZERO_EXTRACT)
	{
	  rtx width = XEXP (SET_DEST (sets[i].rtl), 1);

	  if (CONST_INT_P (src)
	      && CONST_INT_P (width)
	      && INTVAL (width) < HOST_BITS_PER_WIDE_INT
	      && (INTVAL (src) & ((HOST_WIDE_INT) (-1) << INTVAL (width))))
	    src_folded
	      = GEN_INT (INTVAL (src) & ((HOST_WIDE_INT_1
					  << INTVAL (width)) - 1));
	}
#endif

      /* Compute SRC's hash code, and also notice if it
	 should not be recorded at all.  In that case,
	 prevent any further processing of this assignment.

	 We set DO_NOT_RECORD if the destination has a REG_UNUSED note.
	 This avoids getting the source register into the tables, where it
	 may be invalidated later (via REG_QTY), then trigger an ICE upon
	 re-insertion.

	 This is only a problem in multi-set insns.  If it were a single
	 set the dead copy would have been removed.  If the RHS were anything
	 but a simple REG, then we won't call insert_regs and thus there's
	 no potential for triggering the ICE.  */
      do_not_record = (REG_P (dest)
		       && REG_P (src)
		       && find_reg_note (insn, REG_UNUSED, dest));
      hash_arg_in_memory = 0;

      sets[i].src = src;
      sets[i].src_hash = HASH (src, mode);
      sets[i].src_volatile = do_not_record;
      sets[i].src_in_memory = hash_arg_in_memory;

      /* If SRC is a MEM, there is a REG_EQUIV note for SRC, and DEST is
	 a pseudo, do not record SRC.  Using SRC as a replacement for
	 anything else will be incorrect in that situation.  Note that
	 this usually occurs only for stack slots, in which case all the
	 RTL would be referring to SRC, so we don't lose any optimization
	 opportunities by not having SRC in the hash table.  */

      if (MEM_P (src)
	  && find_reg_note (insn, REG_EQUIV, NULL_RTX) != 0
	  && REG_P (dest)
	  && REGNO (dest) >= FIRST_PSEUDO_REGISTER)
	sets[i].src_volatile = 1;

      else if (GET_CODE (src) == ASM_OPERANDS
	       && GET_CODE (x) == PARALLEL)
	{
	  /* Do not record result of a non-volatile inline asm with
	     more than one result.  */
	  if (n_sets > 1)
	    sets[i].src_volatile = 1;

	  int j, lim = XVECLEN (x, 0);
	  for (j = 0; j < lim; j++)
	    {
	      rtx y = XVECEXP (x, 0, j);
	      /* And do not record result of a non-volatile inline asm
		 with "memory" clobber.  */
	      if (GET_CODE (y) == CLOBBER && MEM_P (XEXP (y, 0)))
		{
		  sets[i].src_volatile = 1;
		  break;
		}
	    }
	}

#if 0
      /* It is no longer clear why we used to do this, but it doesn't
	 appear to still be needed.  So let's try without it since this
	 code hurts cse'ing widened ops.  */
      /* If source is a paradoxical subreg (such as QI treated as an SI),
	 treat it as volatile.  It may do the work of an SI in one context
	 where the extra bits are not being used, but cannot replace an SI
	 in general.  */
      if (paradoxical_subreg_p (src))
	sets[i].src_volatile = 1;
#endif

      /* Locate all possible equivalent forms for SRC.  Try to replace
         SRC in the insn with each cheaper equivalent.

         We have the following types of equivalents: SRC itself, a folded
         version, a value given in a REG_EQUAL note, or a value related
	 to a constant.

         Each of these equivalents may be part of an additional class
         of equivalents (if more than one is in the table, they must be in
         the same class; we check for this).

	 If the source is volatile, we don't do any table lookups.

         We note any constant equivalent for possible later use in a
         REG_NOTE.  */

      if (!sets[i].src_volatile)
	elt = lookup (src, sets[i].src_hash, mode);

      sets[i].src_elt = elt;

      if (elt && src_eqv_here && src_eqv_elt)
	{
	  if (elt->first_same_value != src_eqv_elt->first_same_value)
	    {
	      /* The REG_EQUAL is indicating that two formerly distinct
		 classes are now equivalent.  So merge them.  */
	      merge_equiv_classes (elt, src_eqv_elt);
	      src_eqv_hash = HASH (src_eqv, elt->mode);
	      src_eqv_elt = lookup (src_eqv, src_eqv_hash, elt->mode);
	    }

	  src_eqv_here = 0;
	}

      else if (src_eqv_elt)
	elt = src_eqv_elt;

      /* Try to find a constant somewhere and record it in `src_const'.
	 Record its table element, if any, in `src_const_elt'.  Look in
	 any known equivalences first.  (If the constant is not in the
	 table, also set `sets[i].src_const_hash').  */
      if (elt)
	for (p = elt->first_same_value; p; p = p->next_same_value)
	  if (p->is_const)
	    {
	      src_const = p->exp;
	      src_const_elt = elt;
	      break;
	    }

      if (src_const == 0
	  && (CONSTANT_P (src_folded)
	      /* Consider (minus (label_ref L1) (label_ref L2)) as
		 "constant" here so we will record it. This allows us
		 to fold switch statements when an ADDR_DIFF_VEC is used.  */
	      || (GET_CODE (src_folded) == MINUS
		  && GET_CODE (XEXP (src_folded, 0)) == LABEL_REF
		  && GET_CODE (XEXP (src_folded, 1)) == LABEL_REF)))
	src_const = src_folded, src_const_elt = elt;
      else if (src_const == 0 && src_eqv_here && CONSTANT_P (src_eqv_here))
	src_const = src_eqv_here, src_const_elt = src_eqv_elt;

      /* If we don't know if the constant is in the table, get its
	 hash code and look it up.  */
      if (src_const && src_const_elt == 0)
	{
	  sets[i].src_const_hash = HASH (src_const, mode);
	  src_const_elt = lookup (src_const, sets[i].src_const_hash, mode);
	}

      sets[i].src_const = src_const;
      sets[i].src_const_elt = src_const_elt;

      /* If the constant and our source are both in the table, mark them as
	 equivalent.  Otherwise, if a constant is in the table but the source
	 isn't, set ELT to it.  */
      if (src_const_elt && elt
	  && src_const_elt->first_same_value != elt->first_same_value)
	merge_equiv_classes (elt, src_const_elt);
      else if (src_const_elt && elt == 0)
	elt = src_const_elt;

      /* See if there is a register linearly related to a constant
         equivalent of SRC.  */
      if (src_const
	  && (GET_CODE (src_const) == CONST
	      || (src_const_elt && src_const_elt->related_value != 0)))
	{
	  src_related = use_related_value (src_const, src_const_elt);
	  if (src_related)
	    {
	      struct table_elt *src_related_elt
		= lookup (src_related, HASH (src_related, mode), mode);
	      if (src_related_elt && elt)
		{
		  if (elt->first_same_value
		      != src_related_elt->first_same_value)
		    /* This can occur when we previously saw a CONST
		       involving a SYMBOL_REF and then see the SYMBOL_REF
		       twice.  Merge the involved classes.  */
		    merge_equiv_classes (elt, src_related_elt);

		  src_related = 0;
		  src_related_elt = 0;
		}
	      else if (src_related_elt && elt == 0)
		elt = src_related_elt;
	    }
	}

      /* See if we have a CONST_INT that is already in a register in a
	 wider mode.  */

      if (src_const && src_related == 0 && CONST_INT_P (src_const)
	  && is_int_mode (mode, &int_mode)
	  && GET_MODE_PRECISION (int_mode) < BITS_PER_WORD)
	{
	  opt_scalar_int_mode wider_mode_iter;
	  FOR_EACH_WIDER_MODE (wider_mode_iter, int_mode)
	    {
	      scalar_int_mode wider_mode = wider_mode_iter.require ();
	      if (GET_MODE_PRECISION (wider_mode) > BITS_PER_WORD)
		break;

	      struct table_elt *const_elt
		= lookup (src_const, HASH (src_const, wider_mode), wider_mode);

	      if (const_elt == 0)
		continue;

	      for (const_elt = const_elt->first_same_value;
		   const_elt; const_elt = const_elt->next_same_value)
		if (REG_P (const_elt->exp))
		  {
		    src_related = gen_lowpart (int_mode, const_elt->exp);
		    break;
		  }

	      if (src_related != 0)
		break;
	    }
	}

      /* Another possibility is that we have an AND with a constant in
	 a mode narrower than a word.  If so, it might have been generated
	 as part of an "if" which would narrow the AND.  If we already
	 have done the AND in a wider mode, we can use a SUBREG of that
	 value.  */

      if (flag_expensive_optimizations && ! src_related
	  && is_a <scalar_int_mode> (mode, &int_mode)
	  && GET_CODE (src) == AND && CONST_INT_P (XEXP (src, 1))
	  && GET_MODE_SIZE (int_mode) < UNITS_PER_WORD)
	{
	  opt_scalar_int_mode tmode_iter;
	  rtx new_and = gen_rtx_AND (VOIDmode, NULL_RTX, XEXP (src, 1));

	  FOR_EACH_WIDER_MODE (tmode_iter, int_mode)
	    {
	      scalar_int_mode tmode = tmode_iter.require ();
	      if (GET_MODE_SIZE (tmode) > UNITS_PER_WORD)
		break;

	      rtx inner = gen_lowpart (tmode, XEXP (src, 0));
	      struct table_elt *larger_elt;

	      if (inner)
		{
		  PUT_MODE (new_and, tmode);
		  XEXP (new_and, 0) = inner;
		  larger_elt = lookup (new_and, HASH (new_and, tmode), tmode);
		  if (larger_elt == 0)
		    continue;

		  for (larger_elt = larger_elt->first_same_value;
		       larger_elt; larger_elt = larger_elt->next_same_value)
		    if (REG_P (larger_elt->exp))
		      {
			src_related
			  = gen_lowpart (int_mode, larger_elt->exp);
			break;
		      }

		  if (src_related)
		    break;
		}
	    }
	}

      /* See if a MEM has already been loaded with a widening operation;
	 if it has, we can use a subreg of that.  Many CISC machines
	 also have such operations, but this is only likely to be
	 beneficial on these machines.  */

      rtx_code extend_op;
      if (flag_expensive_optimizations && src_related == 0
	  && MEM_P (src) && ! do_not_record
	  && is_a <scalar_int_mode> (mode, &int_mode)
	  && (extend_op = load_extend_op (int_mode)) != UNKNOWN)
	{
	  struct rtx_def memory_extend_buf;
	  rtx memory_extend_rtx = &memory_extend_buf;

	  /* Set what we are trying to extend and the operation it might
	     have been extended with.  */
	  memset (memory_extend_rtx, 0, sizeof (*memory_extend_rtx));
	  PUT_CODE (memory_extend_rtx, extend_op);
	  XEXP (memory_extend_rtx, 0) = src;

	  opt_scalar_int_mode tmode_iter;
	  FOR_EACH_WIDER_MODE (tmode_iter, int_mode)
	    {
	      struct table_elt *larger_elt;

	      scalar_int_mode tmode = tmode_iter.require ();
	      if (GET_MODE_SIZE (tmode) > UNITS_PER_WORD)
		break;

	      PUT_MODE (memory_extend_rtx, tmode);
	      larger_elt = lookup (memory_extend_rtx,
				   HASH (memory_extend_rtx, tmode), tmode);
	      if (larger_elt == 0)
		continue;

	      for (larger_elt = larger_elt->first_same_value;
		   larger_elt; larger_elt = larger_elt->next_same_value)
		if (REG_P (larger_elt->exp))
		  {
		    src_related = gen_lowpart (int_mode, larger_elt->exp);
		    break;
		  }

	      if (src_related)
		break;
	    }
	}

      /* Try to express the constant using a register+offset expression
	 derived from a constant anchor.  */

      if (targetm.const_anchor
	  && !src_related
	  && src_const
	  && GET_CODE (src_const) == CONST_INT)
	{
	  src_related = try_const_anchors (src_const, mode);
	  src_related_is_const_anchor = src_related != NULL_RTX;
	}

      /* Try to re-materialize a vec_dup with an existing constant.   */
      rtx src_elt;
      if ((!src_eqv_here || CONSTANT_P (src_eqv_here))
	  && const_vec_duplicate_p (src, &src_elt))
	{
	   machine_mode const_mode = GET_MODE_INNER (GET_MODE (src));
	   struct table_elt *related_elt
		= lookup (src_elt, HASH (src_elt, const_mode), const_mode);
	   if (related_elt)
	    {
	      for (related_elt = related_elt->first_same_value;
		   related_elt; related_elt = related_elt->next_same_value)
		if (REG_P (related_elt->exp))
		  {
		   /* We don't need to compare costs with an existing (constant)
		      src_eqv_here, since any such src_eqv_here should already be
		      available in src_const.  */
		    src_eqv_here
			= gen_rtx_VEC_DUPLICATE (GET_MODE (src),
						 related_elt->exp);
		    break;
		  }
	    }
	}

      if (src == src_folded)
	src_folded = 0;

      /* At this point, ELT, if nonzero, points to a class of expressions
         equivalent to the source of this SET and SRC, SRC_EQV, SRC_FOLDED,
	 and SRC_RELATED, if nonzero, each contain additional equivalent
	 expressions.  Prune these latter expressions by deleting expressions
	 already in the equivalence class.

	 Check for an equivalent identical to the destination.  If found,
	 this is the preferred equivalent since it will likely lead to
	 elimination of the insn.  Indicate this by placing it in
	 `src_related'.  */

      if (elt)
	elt = elt->first_same_value;
      for (p = elt; p; p = p->next_same_value)
	{
	  enum rtx_code code = GET_CODE (p->exp);

	  /* If the expression is not valid, ignore it.  Then we do not
	     have to check for validity below.  In most cases, we can use
	     `rtx_equal_p', since canonicalization has already been done.  */
	  if (code != REG && ! exp_equiv_p (p->exp, p->exp, 1, false))
	    continue;

	  /* Also skip paradoxical subregs, unless that's what we're
	     looking for.  */
	  if (paradoxical_subreg_p (p->exp)
	      && ! (src != 0
		    && GET_CODE (src) == SUBREG
		    && GET_MODE (src) == GET_MODE (p->exp)
		    && partial_subreg_p (GET_MODE (SUBREG_REG (src)),
					 GET_MODE (SUBREG_REG (p->exp)))))
	    continue;

	  if (src && GET_CODE (src) == code && rtx_equal_p (src, p->exp))
	    src = 0;
	  else if (src_folded && GET_CODE (src_folded) == code
		   && rtx_equal_p (src_folded, p->exp))
	    src_folded = 0;
	  else if (src_eqv_here && GET_CODE (src_eqv_here) == code
		   && rtx_equal_p (src_eqv_here, p->exp))
	    src_eqv_here = 0;
	  else if (src_related && GET_CODE (src_related) == code
		   && rtx_equal_p (src_related, p->exp))
	    src_related = 0;

	  /* This is the same as the destination of the insns, we want
	     to prefer it.  Copy it to src_related.  The code below will
	     then give it a negative cost.  */
	  if (GET_CODE (dest) == code && rtx_equal_p (p->exp, dest))
	    src_related = p->exp;
	}

      /* Find the cheapest valid equivalent, trying all the available
         possibilities.  Prefer items not in the hash table to ones
         that are when they are equal cost.  Note that we can never
         worsen an insn as the current contents will also succeed.
	 If we find an equivalent identical to the destination, use it as best,
	 since this insn will probably be eliminated in that case.  */
      if (src)
	{
	  if (rtx_equal_p (src, dest))
	    src_cost = src_regcost = -1;
	  else
	    {
	      src_cost = COST (src, mode);
	      src_regcost = approx_reg_cost (src);
	    }
	}

      if (src_eqv_here)
	{
	  if (rtx_equal_p (src_eqv_here, dest))
	    src_eqv_cost = src_eqv_regcost = -1;
	  else
	    {
	      src_eqv_cost = COST (src_eqv_here, mode);
	      src_eqv_regcost = approx_reg_cost (src_eqv_here);
	    }
	}

      if (src_folded)
	{
	  if (rtx_equal_p (src_folded, dest))
	    src_folded_cost = src_folded_regcost = -1;
	  else
	    {
	      src_folded_cost = COST (src_folded, mode);
	      src_folded_regcost = approx_reg_cost (src_folded);
	    }
	}

      if (src_related)
	{
	  if (rtx_equal_p (src_related, dest))
	    src_related_cost = src_related_regcost = -1;
	  else
	    {
	      src_related_cost = COST (src_related, mode);
	      src_related_regcost = approx_reg_cost (src_related);

	      /* If a const-anchor is used to synthesize a constant that
		 normally requires multiple instructions then slightly prefer
		 it over the original sequence.  These instructions are likely
		 to become redundant now.  We can't compare against the cost
		 of src_eqv_here because, on MIPS for example, multi-insn
		 constants have zero cost; they are assumed to be hoisted from
		 loops.  */
	      if (src_related_is_const_anchor
		  && src_related_cost == src_cost
		  && src_eqv_here)
		src_related_cost--;
	    }
	}

      /* If this was an indirect jump insn, a known label will really be
	 cheaper even though it looks more expensive.  */
      if (dest == pc_rtx && src_const && GET_CODE (src_const) == LABEL_REF)
	src_folded = src_const, src_folded_cost = src_folded_regcost = -1;

      /* Terminate loop when replacement made.  This must terminate since
         the current contents will be tested and will always be valid.  */
      while (1)
	{
	  rtx trial;

	  /* Skip invalid entries.  */
	  while (elt && !REG_P (elt->exp)
		 && ! exp_equiv_p (elt->exp, elt->exp, 1, false))
	    elt = elt->next_same_value;

	  /* A paradoxical subreg would be bad here: it'll be the right
	     size, but later may be adjusted so that the upper bits aren't
	     what we want.  So reject it.  */
	  if (elt != 0
	      && paradoxical_subreg_p (elt->exp)
	      /* It is okay, though, if the rtx we're trying to match
		 will ignore any of the bits we can't predict.  */
	      && ! (src != 0
		    && GET_CODE (src) == SUBREG
		    && GET_MODE (src) == GET_MODE (elt->exp)
		    && partial_subreg_p (GET_MODE (SUBREG_REG (src)),
					 GET_MODE (SUBREG_REG (elt->exp)))))
	    {
	      elt = elt->next_same_value;
	      continue;
	    }

	  if (elt)
	    {
	      src_elt_cost = elt->cost;
	      src_elt_regcost = elt->regcost;
	    }

	  /* Find cheapest and skip it for the next time.   For items
	     of equal cost, use this order:
	     src_folded, src, src_eqv, src_related and hash table entry.  */
	  if (src_folded
	      && preferable (src_folded_cost, src_folded_regcost,
			     src_cost, src_regcost) <= 0
	      && preferable (src_folded_cost, src_folded_regcost,
			     src_eqv_cost, src_eqv_regcost) <= 0
	      && preferable (src_folded_cost, src_folded_regcost,
			     src_related_cost, src_related_regcost) <= 0
	      && preferable (src_folded_cost, src_folded_regcost,
			     src_elt_cost, src_elt_regcost) <= 0)
	    trial = src_folded, src_folded_cost = MAX_COST;
	  else if (src
		   && preferable (src_cost, src_regcost,
				  src_eqv_cost, src_eqv_regcost) <= 0
		   && preferable (src_cost, src_regcost,
				  src_related_cost, src_related_regcost) <= 0
		   && preferable (src_cost, src_regcost,
				  src_elt_cost, src_elt_regcost) <= 0)
	    trial = src, src_cost = MAX_COST;
	  else if (src_eqv_here
		   && preferable (src_eqv_cost, src_eqv_regcost,
				  src_related_cost, src_related_regcost) <= 0
		   && preferable (src_eqv_cost, src_eqv_regcost,
				  src_elt_cost, src_elt_regcost) <= 0)
	    trial = src_eqv_here, src_eqv_cost = MAX_COST;
	  else if (src_related
		   && preferable (src_related_cost, src_related_regcost,
				  src_elt_cost, src_elt_regcost) <= 0)
	    trial = src_related, src_related_cost = MAX_COST;
	  else
	    {
	      trial = elt->exp;
	      elt = elt->next_same_value;
	      src_elt_cost = MAX_COST;
	    }

	  /* Try to optimize
	     (set (reg:M N) (const_int A))
	     (set (reg:M2 O) (const_int B))
	     (set (zero_extract:M2 (reg:M N) (const_int C) (const_int D))
		  (reg:M2 O)).  */
	  if (GET_CODE (SET_DEST (sets[i].rtl)) == ZERO_EXTRACT
	      && CONST_INT_P (trial)
	      && CONST_INT_P (XEXP (SET_DEST (sets[i].rtl), 1))
	      && CONST_INT_P (XEXP (SET_DEST (sets[i].rtl), 2))
	      && REG_P (XEXP (SET_DEST (sets[i].rtl), 0))
	      && (known_ge
		  (GET_MODE_PRECISION (GET_MODE (SET_DEST (sets[i].rtl))),
		   INTVAL (XEXP (SET_DEST (sets[i].rtl), 1))))
	      && ((unsigned) INTVAL (XEXP (SET_DEST (sets[i].rtl), 1))
		  + (unsigned) INTVAL (XEXP (SET_DEST (sets[i].rtl), 2))
		  <= HOST_BITS_PER_WIDE_INT))
	    {
	      rtx dest_reg = XEXP (SET_DEST (sets[i].rtl), 0);
	      rtx width = XEXP (SET_DEST (sets[i].rtl), 1);
	      rtx pos = XEXP (SET_DEST (sets[i].rtl), 2);
	      unsigned int dest_hash = HASH (dest_reg, GET_MODE (dest_reg));
	      struct table_elt *dest_elt
		= lookup (dest_reg, dest_hash, GET_MODE (dest_reg));
	      rtx dest_cst = NULL;

	      if (dest_elt)
		for (p = dest_elt->first_same_value; p; p = p->next_same_value)
		  if (p->is_const && CONST_INT_P (p->exp))
		    {
		      dest_cst = p->exp;
		      break;
		    }
	      if (dest_cst)
		{
		  HOST_WIDE_INT val = INTVAL (dest_cst);
		  HOST_WIDE_INT mask;
		  unsigned int shift;
		  /* This is the mode of DEST_CST as well.  */
		  scalar_int_mode dest_mode
		    = as_a <scalar_int_mode> (GET_MODE (dest_reg));
		  if (BITS_BIG_ENDIAN)
		    shift = GET_MODE_PRECISION (dest_mode)
			    - INTVAL (pos) - INTVAL (width);
		  else
		    shift = INTVAL (pos);
		  if (INTVAL (width) == HOST_BITS_PER_WIDE_INT)
		    mask = HOST_WIDE_INT_M1;
		  else
		    mask = (HOST_WIDE_INT_1 << INTVAL (width)) - 1;
		  val &= ~(mask << shift);
		  val |= (INTVAL (trial) & mask) << shift;
		  val = trunc_int_for_mode (val, dest_mode);
		  validate_unshare_change (insn, &SET_DEST (sets[i].rtl),
					   dest_reg, 1);
		  validate_unshare_change (insn, &SET_SRC (sets[i].rtl),
					   GEN_INT (val), 1);
		  if (apply_change_group ())
		    {
		      rtx note = find_reg_note (insn, REG_EQUAL, NULL_RTX);
		      if (note)
			{
			  remove_note (insn, note);
			  df_notes_rescan (insn);
			}
		      src_eqv = NULL_RTX;
		      src_eqv_elt = NULL;
		      src_eqv_volatile = 0;
		      src_eqv_in_memory = 0;
		      src_eqv_hash = 0;
		      repeat = true;
		      break;
		    }
		}
	    }

	  /* We don't normally have an insn matching (set (pc) (pc)), so
	     check for this separately here.  We will delete such an
	     insn below.

	     For other cases such as a table jump or conditional jump
	     where we know the ultimate target, go ahead and replace the
	     operand.  While that may not make a valid insn, we will
	     reemit the jump below (and also insert any necessary
	     barriers).  */
	  if (n_sets == 1 && dest == pc_rtx
	      && (trial == pc_rtx
		  || (GET_CODE (trial) == LABEL_REF
		      && ! condjump_p (insn))))
	    {
	      /* Don't substitute non-local labels, this confuses CFG.  */
	      if (GET_CODE (trial) == LABEL_REF
		  && LABEL_REF_NONLOCAL_P (trial))
		continue;

	      SET_SRC (sets[i].rtl) = trial;
	      cse_jumps_altered = true;
	      break;
	    }

	  /* Similarly, lots of targets don't allow no-op
	     (set (mem x) (mem x)) moves.  Even (set (reg x) (reg x))
	     might be impossible for certain registers (like CC registers).  */
	  else if (n_sets == 1
		   && !CALL_P (insn)
		   && (MEM_P (trial) || REG_P (trial))
		   && rtx_equal_p (trial, dest)
		   && !side_effects_p (dest)
		   && (cfun->can_delete_dead_exceptions
		       || insn_nothrow_p (insn))
		   /* We can only remove the later store if the earlier aliases
		      at least all accesses the later one.  */
		   && (!MEM_P (trial)
		       || ((MEM_ALIAS_SET (dest) == MEM_ALIAS_SET (trial)
			    || alias_set_subset_of (MEM_ALIAS_SET (dest),
						    MEM_ALIAS_SET (trial)))
			    && (!MEM_EXPR (trial)
				|| refs_same_for_tbaa_p (MEM_EXPR (trial),
							 MEM_EXPR (dest))))))
	    {
	      SET_SRC (sets[i].rtl) = trial;
	      noop_insn = true;
	      break;
	    }

	  /* Reject certain invalid forms of CONST that we create.  */
	  else if (CONSTANT_P (trial)
		   && GET_CODE (trial) == CONST
		   /* Reject cases that will cause decode_rtx_const to
		      die.  On the alpha when simplifying a switch, we
		      get (const (truncate (minus (label_ref)
		      (label_ref)))).  */
		   && (GET_CODE (XEXP (trial, 0)) == TRUNCATE
		       /* Likewise on IA-64, except without the
			  truncate.  */
		       || (GET_CODE (XEXP (trial, 0)) == MINUS
			   && GET_CODE (XEXP (XEXP (trial, 0), 0)) == LABEL_REF
			   && GET_CODE (XEXP (XEXP (trial, 0), 1)) == LABEL_REF)))
	    /* Do nothing for this case.  */
	    ;

	  /* Do not replace anything with a MEM, except the replacement
	     is a no-op.  This allows this loop to terminate.  */
	  else if (MEM_P (trial) && !rtx_equal_p (trial, SET_SRC(sets[i].rtl)))
	    /* Do nothing for this case.  */
	    ;

	  /* Look for a substitution that makes a valid insn.  */
	  else if (validate_unshare_change (insn, &SET_SRC (sets[i].rtl),
					    trial, 0))
	    {
	      rtx new_rtx = canon_reg (SET_SRC (sets[i].rtl), insn);

	      /* The result of apply_change_group can be ignored; see
		 canon_reg.  */

	      validate_change (insn, &SET_SRC (sets[i].rtl), new_rtx, 1);
	      apply_change_group ();

	      break;
	    }

	  /* If the current function uses a constant pool and this is a
	     constant, try making a pool entry. Put it in src_folded
	     unless we already have done this since that is where it
	     likely came from.  */

	  else if (crtl->uses_const_pool
		   && CONSTANT_P (trial)
		   && !CONST_INT_P (trial)
		   && (src_folded == 0 || !MEM_P (src_folded))
		   && GET_MODE_CLASS (mode) != MODE_CC
		   && mode != VOIDmode)
	    {
	      src_folded = force_const_mem (mode, trial);
	      if (src_folded)
		{
		  src_folded_cost = COST (src_folded, mode);
		  src_folded_regcost = approx_reg_cost (src_folded);
		}
	    }
	}

      /* If we changed the insn too much, handle this set from scratch.  */
      if (repeat)
	{
	  i--;
	  continue;
	}

      src = SET_SRC (sets[i].rtl);

      /* In general, it is good to have a SET with SET_SRC == SET_DEST.
	 However, there is an important exception:  If both are registers
	 that are not the head of their equivalence class, replace SET_SRC
	 with the head of the class.  If we do not do this, we will have
	 both registers live over a portion of the basic block.  This way,
	 their lifetimes will likely abut instead of overlapping.  */
      if (REG_P (dest)
	  && REGNO_QTY_VALID_P (REGNO (dest)))
	{
	  int dest_q = REG_QTY (REGNO (dest));
	  struct qty_table_elem *dest_ent = &qty_table[dest_q];

	  if (dest_ent->mode == GET_MODE (dest)
	      && dest_ent->first_reg != REGNO (dest)
	      && REG_P (src) && REGNO (src) == REGNO (dest)
	      /* Don't do this if the original insn had a hard reg as
		 SET_SRC or SET_DEST.  */
	      && (!REG_P (sets[i].src)
		  || REGNO (sets[i].src) >= FIRST_PSEUDO_REGISTER)
	      && (!REG_P (dest) || REGNO (dest) >= FIRST_PSEUDO_REGISTER))
	    /* We can't call canon_reg here because it won't do anything if
	       SRC is a hard register.  */
	    {
	      int src_q = REG_QTY (REGNO (src));
	      struct qty_table_elem *src_ent = &qty_table[src_q];
	      int first = src_ent->first_reg;
	      rtx new_src
		= (first >= FIRST_PSEUDO_REGISTER
		   ? regno_reg_rtx[first] : gen_rtx_REG (GET_MODE (src), first));

	      /* We must use validate-change even for this, because this
		 might be a special no-op instruction, suitable only to
		 tag notes onto.  */
	      if (validate_change (insn, &SET_SRC (sets[i].rtl), new_src, 0))
		{
		  src = new_src;
		  /* If we had a constant that is cheaper than what we are now
		     setting SRC to, use that constant.  We ignored it when we
		     thought we could make this into a no-op.  */
		  if (src_const && COST (src_const, mode) < COST (src, mode)
		      && validate_change (insn, &SET_SRC (sets[i].rtl),
					  src_const, 0))
		    src = src_const;
		}
	    }
	}

      /* If we made a change, recompute SRC values.  */
      if (src != sets[i].src)
	{
	  do_not_record = 0;
	  hash_arg_in_memory = 0;
	  sets[i].src = src;
	  sets[i].src_hash = HASH (src, mode);
	  sets[i].src_volatile = do_not_record;
	  sets[i].src_in_memory = hash_arg_in_memory;
	  sets[i].src_elt = lookup (src, sets[i].src_hash, mode);
	}

      /* If this is a single SET, we are setting a register, and we have an
	 equivalent constant, we want to add a REG_EQUAL note if the constant
	 is different from the source.  We don't want to do it for a constant
	 pseudo since verifying that this pseudo hasn't been eliminated is a
	 pain; moreover such a note won't help anything.

	 Avoid a REG_EQUAL note for (CONST (MINUS (LABEL_REF) (LABEL_REF)))
	 which can be created for a reference to a compile time computable
	 entry in a jump table.  */
      if (n_sets == 1
	  && REG_P (dest)
	  && src_const
	  && !REG_P (src_const)
	  && !(GET_CODE (src_const) == SUBREG
	       && REG_P (SUBREG_REG (src_const)))
	  && !(GET_CODE (src_const) == CONST
	       && GET_CODE (XEXP (src_const, 0)) == MINUS
	       && GET_CODE (XEXP (XEXP (src_const, 0), 0)) == LABEL_REF
	       && GET_CODE (XEXP (XEXP (src_const, 0), 1)) == LABEL_REF)
	  && !rtx_equal_p (src, src_const))
	{
	  /* Make sure that the rtx is not shared.  */
	  src_const = copy_rtx (src_const);

	  /* Record the actual constant value in a REG_EQUAL note,
	     making a new one if one does not already exist.  */
	  set_unique_reg_note (insn, REG_EQUAL, src_const);
	  df_notes_rescan (insn);
	}

      /* Now deal with the destination.  */
      do_not_record = 0;

      /* Look within any ZERO_EXTRACT to the MEM or REG within it.  */
      while (GET_CODE (dest) == SUBREG
	     || GET_CODE (dest) == ZERO_EXTRACT
	     || GET_CODE (dest) == STRICT_LOW_PART)
	dest = XEXP (dest, 0);

      sets[i].inner_dest = dest;

      if (MEM_P (dest))
	{
#ifdef PUSH_ROUNDING
	  /* Stack pushes invalidate the stack pointer.  */
	  rtx addr = XEXP (dest, 0);
	  if (GET_RTX_CLASS (GET_CODE (addr)) == RTX_AUTOINC
	      && XEXP (addr, 0) == stack_pointer_rtx)
	    invalidate (stack_pointer_rtx, VOIDmode);
#endif
	  dest = fold_rtx (dest, insn);
	}

      /* Compute the hash code of the destination now,
	 before the effects of this instruction are recorded,
	 since the register values used in the address computation
	 are those before this instruction.  */
      sets[i].dest_hash = HASH (dest, mode);

      /* Don't enter a bit-field in the hash table
	 because the value in it after the store
	 may not equal what was stored, due to truncation.  */

      if (GET_CODE (SET_DEST (sets[i].rtl)) == ZERO_EXTRACT)
	{
	  rtx width = XEXP (SET_DEST (sets[i].rtl), 1);

	  if (src_const != 0 && CONST_INT_P (src_const)
	      && CONST_INT_P (width)
	      && INTVAL (width) < HOST_BITS_PER_WIDE_INT
	      && ! (INTVAL (src_const)
		    & (HOST_WIDE_INT_M1U << INTVAL (width))))
	    /* Exception: if the value is constant,
	       and it won't be truncated, record it.  */
	    ;
	  else
	    {
	      /* This is chosen so that the destination will be invalidated
		 but no new value will be recorded.
		 We must invalidate because sometimes constant
		 values can be recorded for bitfields.  */
	      sets[i].src_elt = 0;
	      sets[i].src_volatile = 1;
	      src_eqv = 0;
	      src_eqv_elt = 0;
	    }
	}

      /* If only one set in a JUMP_INSN and it is now a no-op, we can delete
	 the insn.  */
      else if (n_sets == 1 && dest == pc_rtx && src == pc_rtx)
	{
	  /* One less use of the label this insn used to jump to.  */
	  cse_cfg_altered |= delete_insn_and_edges (insn);
	  cse_jumps_altered = true;
	  /* No more processing for this set.  */
	  sets[i].rtl = 0;
	}

      /* Similarly for no-op moves.  */
      else if (noop_insn)
	{
	  if (cfun->can_throw_non_call_exceptions && can_throw_internal (insn))
	    cse_cfg_altered = true;
	  cse_cfg_altered |= delete_insn_and_edges (insn);
	  /* No more processing for this set.  */
	  sets[i].rtl = 0;
	}

      /* If this SET is now setting PC to a label, we know it used to
	 be a conditional or computed branch.  */
      else if (dest == pc_rtx && GET_CODE (src) == LABEL_REF
	       && !LABEL_REF_NONLOCAL_P (src))
	{
	  /* We reemit the jump in as many cases as possible just in
	     case the form of an unconditional jump is significantly
	     different than a computed jump or conditional jump.

	     If this insn has multiple sets, then reemitting the
	     jump is nontrivial.  So instead we just force rerecognition
	     and hope for the best.  */
	  if (n_sets == 1)
	    {
	      rtx_jump_insn *new_rtx;
	      rtx note;

	      rtx_insn *seq = targetm.gen_jump (XEXP (src, 0));
	      new_rtx = emit_jump_insn_before (seq, insn);
	      JUMP_LABEL (new_rtx) = XEXP (src, 0);
	      LABEL_NUSES (XEXP (src, 0))++;

	      /* Make sure to copy over REG_NON_LOCAL_GOTO.  */
	      note = find_reg_note (insn, REG_NON_LOCAL_GOTO, 0);
	      if (note)
		{
		  XEXP (note, 1) = NULL_RTX;
		  REG_NOTES (new_rtx) = note;
		}

	      cse_cfg_altered |= delete_insn_and_edges (insn);
	      insn = new_rtx;
	    }
	  else
	    INSN_CODE (insn) = -1;

	  /* Do not bother deleting any unreachable code, let jump do it.  */
	  cse_jumps_altered = true;
	  sets[i].rtl = 0;
	}

      /* If destination is volatile, invalidate it and then do no further
	 processing for this assignment.  */

      else if (do_not_record)
	{
	  invalidate_dest (dest);
	  sets[i].rtl = 0;
	}

      if (sets[i].rtl != 0 && dest != SET_DEST (sets[i].rtl))
	{
	  do_not_record = 0;
	  sets[i].dest_hash = HASH (SET_DEST (sets[i].rtl), mode);
	  if (do_not_record)
	    {
	      invalidate_dest (SET_DEST (sets[i].rtl));
	      sets[i].rtl = 0;
	    }
	}
    }

  /* Now enter all non-volatile source expressions in the hash table
     if they are not already present.
     Record their equivalence classes in src_elt.
     This way we can insert the corresponding destinations into
     the same classes even if the actual sources are no longer in them
     (having been invalidated).  */

  if (src_eqv && src_eqv_elt == 0 && sets[0].rtl != 0 && ! src_eqv_volatile
      && ! rtx_equal_p (src_eqv, SET_DEST (sets[0].rtl)))
    {
      struct table_elt *elt;
      struct table_elt *classp = sets[0].src_elt;
      rtx dest = SET_DEST (sets[0].rtl);
      machine_mode eqvmode = GET_MODE (dest);

      if (GET_CODE (dest) == STRICT_LOW_PART)
	{
	  eqvmode = GET_MODE (SUBREG_REG (XEXP (dest, 0)));
	  classp = 0;
	}
      if (insert_regs (src_eqv, classp, 0))
	{
	  rehash_using_reg (src_eqv);
	  src_eqv_hash = HASH (src_eqv, eqvmode);
	}
      elt = insert (src_eqv, classp, src_eqv_hash, eqvmode);
      elt->in_memory = src_eqv_in_memory;
      src_eqv_elt = elt;

      /* Check to see if src_eqv_elt is the same as a set source which
	 does not yet have an elt, and if so set the elt of the set source
	 to src_eqv_elt.  */
      for (i = 0; i < n_sets; i++)
	if (sets[i].rtl && sets[i].src_elt == 0
	    && rtx_equal_p (SET_SRC (sets[i].rtl), src_eqv))
	  sets[i].src_elt = src_eqv_elt;
    }

  for (i = 0; i < n_sets; i++)
    if (sets[i].rtl && ! sets[i].src_volatile
	&& ! rtx_equal_p (SET_SRC (sets[i].rtl), SET_DEST (sets[i].rtl)))
      {
	if (GET_CODE (SET_DEST (sets[i].rtl)) == STRICT_LOW_PART)
	  {
	    /* REG_EQUAL in setting a STRICT_LOW_PART
	       gives an equivalent for the entire destination register,
	       not just for the subreg being stored in now.
	       This is a more interesting equivalence, so we arrange later
	       to treat the entire reg as the destination.  */
	    sets[i].src_elt = src_eqv_elt;
	    sets[i].src_hash = src_eqv_hash;
	  }
	else
	  {
	    /* Insert source and constant equivalent into hash table, if not
	       already present.  */
	    struct table_elt *classp = src_eqv_elt;
	    rtx src = sets[i].src;
	    rtx dest = SET_DEST (sets[i].rtl);
	    machine_mode mode
	      = GET_MODE (src) == VOIDmode ? GET_MODE (dest) : GET_MODE (src);

	    /* It's possible that we have a source value known to be
	       constant but don't have a REG_EQUAL note on the insn.
	       Lack of a note will mean src_eqv_elt will be NULL.  This
	       can happen where we've generated a SUBREG to access a
	       CONST_INT that is already in a register in a wider mode.
	       Ensure that the source expression is put in the proper
	       constant class.  */
	    if (!classp)
	      classp = sets[i].src_const_elt;

	    if (sets[i].src_elt == 0)
	      {
		struct table_elt *elt;

		/* Note that these insert_regs calls cannot remove
		   any of the src_elt's, because they would have failed to
		   match if not still valid.  */
		if (insert_regs (src, classp, 0))
		  {
		    rehash_using_reg (src);
		    sets[i].src_hash = HASH (src, mode);
		  }
		elt = insert (src, classp, sets[i].src_hash, mode);
		elt->in_memory = sets[i].src_in_memory;
		/* If inline asm has any clobbers, ensure we only reuse
		   existing inline asms and never try to put the ASM_OPERANDS
		   into an insn that isn't inline asm.  */
		if (GET_CODE (src) == ASM_OPERANDS
		    && GET_CODE (x) == PARALLEL)
		  elt->cost = MAX_COST;
		sets[i].src_elt = classp = elt;
	      }
	    if (sets[i].src_const && sets[i].src_const_elt == 0
		&& src != sets[i].src_const
		&& ! rtx_equal_p (sets[i].src_const, src))
	      sets[i].src_elt = insert (sets[i].src_const, classp,
					sets[i].src_const_hash, mode);
	  }
      }
    else if (sets[i].src_elt == 0)
      /* If we did not insert the source into the hash table (e.g., it was
	 volatile), note the equivalence class for the REG_EQUAL value, if any,
	 so that the destination goes into that class.  */
      sets[i].src_elt = src_eqv_elt;

  /* Record destination addresses in the hash table.  This allows us to
     check if they are invalidated by other sets.  */
  for (i = 0; i < n_sets; i++)
    {
      if (sets[i].rtl)
	{
	  rtx x = sets[i].inner_dest;
	  struct table_elt *elt;
	  machine_mode mode;
	  unsigned hash;

	  if (MEM_P (x))
	    {
	      x = XEXP (x, 0);
	      mode = GET_MODE (x);
	      hash = HASH (x, mode);
	      elt = lookup (x, hash, mode);
	      if (!elt)
		{
		  if (insert_regs (x, NULL, 0))
		    {
		      rtx dest = SET_DEST (sets[i].rtl);

		      rehash_using_reg (x);
		      hash = HASH (x, mode);
		      sets[i].dest_hash = HASH (dest, GET_MODE (dest));
		    }
		  elt = insert (x, NULL, hash, mode);
		}

	      sets[i].dest_addr_elt = elt;
	    }
	  else
	    sets[i].dest_addr_elt = NULL;
	}
    }

  invalidate_from_clobbers (insn);

  /* Some registers are invalidated by subroutine calls.  Memory is
     invalidated by non-constant calls.  */

  if (CALL_P (insn))
    {
      if (!(RTL_CONST_OR_PURE_CALL_P (insn)))
	invalidate_memory ();
      else
	/* For const/pure calls, invalidate any argument slots, because
	   those are owned by the callee.  */
	for (tem = CALL_INSN_FUNCTION_USAGE (insn); tem; tem = XEXP (tem, 1))
	  if (GET_CODE (XEXP (tem, 0)) == USE
	      && MEM_P (XEXP (XEXP (tem, 0), 0)))
	    invalidate (XEXP (XEXP (tem, 0), 0), VOIDmode);
      invalidate_for_call (insn);
    }

  /* Now invalidate everything set by this instruction.
     If a SUBREG or other funny destination is being set,
     sets[i].rtl is still nonzero, so here we invalidate the reg
     a part of which is being set.  */

  for (i = 0; i < n_sets; i++)
    if (sets[i].rtl)
      {
	/* We can't use the inner dest, because the mode associated with
	   a ZERO_EXTRACT is significant.  */
	rtx dest = SET_DEST (sets[i].rtl);

	/* Needed for registers to remove the register from its
	   previous quantity's chain.
	   Needed for memory if this is a nonvarying address, unless
	   we have just done an invalidate_memory that covers even those.  */
	if (REG_P (dest) || GET_CODE (dest) == SUBREG)
	  invalidate (dest, VOIDmode);
	else if (MEM_P (dest))
	  invalidate (dest, VOIDmode);
	else if (GET_CODE (dest) == STRICT_LOW_PART
		 || GET_CODE (dest) == ZERO_EXTRACT)
	  invalidate (XEXP (dest, 0), GET_MODE (dest));
      }

  /* Don't cse over a call to setjmp; on some machines (eg VAX)
     the regs restored by the longjmp come from a later time
     than the setjmp.  */
  if (CALL_P (insn) && find_reg_note (insn, REG_SETJMP, NULL))
    {
      flush_hash_table ();
      goto done;
    }

  /* Make sure registers mentioned in destinations
     are safe for use in an expression to be inserted.
     This removes from the hash table
     any invalid entry that refers to one of these registers.

     We don't care about the return value from mention_regs because
     we are going to hash the SET_DEST values unconditionally.  */

  for (i = 0; i < n_sets; i++)
    {
      if (sets[i].rtl)
	{
	  rtx x = SET_DEST (sets[i].rtl);

	  if (!REG_P (x))
	    mention_regs (x);
	  else
	    {
	      /* We used to rely on all references to a register becoming
		 inaccessible when a register changes to a new quantity,
		 since that changes the hash code.  However, that is not
		 safe, since after HASH_SIZE new quantities we get a
		 hash 'collision' of a register with its own invalid
		 entries.  And since SUBREGs have been changed not to
		 change their hash code with the hash code of the register,
		 it wouldn't work any longer at all.  So we have to check
		 for any invalid references lying around now.
		 This code is similar to the REG case in mention_regs,
		 but it knows that reg_tick has been incremented, and
		 it leaves reg_in_table as -1 .  */
	      unsigned int regno = REGNO (x);
	      unsigned int endregno = END_REGNO (x);
	      unsigned int i;

	      for (i = regno; i < endregno; i++)
		{
		  if (REG_IN_TABLE (i) >= 0)
		    {
		      remove_invalid_refs (i);
		      REG_IN_TABLE (i) = -1;
		    }
		}
	    }
	}
    }

  /* We may have just removed some of the src_elt's from the hash table.
     So replace each one with the current head of the same class.
     Also check if destination addresses have been removed.  */

  for (i = 0; i < n_sets; i++)
    if (sets[i].rtl)
      {
	if (sets[i].dest_addr_elt
	    && sets[i].dest_addr_elt->first_same_value == 0)
	  {
	    /* The elt was removed, which means this destination is not
	       valid after this instruction.  */
	    sets[i].rtl = NULL_RTX;
	  }
	else if (sets[i].src_elt && sets[i].src_elt->first_same_value == 0)
	  /* If elt was removed, find current head of same class,
	     or 0 if nothing remains of that class.  */
	  {
	    struct table_elt *elt = sets[i].src_elt;

	    while (elt && elt->prev_same_value)
	      elt = elt->prev_same_value;

	    while (elt && elt->first_same_value == 0)
	      elt = elt->next_same_value;
	    sets[i].src_elt = elt ? elt->first_same_value : 0;
	  }
      }

  /* Now insert the destinations into their equivalence classes.  */

  for (i = 0; i < n_sets; i++)
    if (sets[i].rtl)
      {
	rtx dest = SET_DEST (sets[i].rtl);
	struct table_elt *elt;

	/* Don't record value if we are not supposed to risk allocating
	   floating-point values in registers that might be wider than
	   memory.  */
	if ((flag_float_store
	     && MEM_P (dest)
	     && FLOAT_MODE_P (GET_MODE (dest)))
	    /* Don't record BLKmode values, because we don't know the
	       size of it, and can't be sure that other BLKmode values
	       have the same or smaller size.  */
	    || GET_MODE (dest) == BLKmode
	    /* If we didn't put a REG_EQUAL value or a source into the hash
	       table, there is no point is recording DEST.  */
	    || sets[i].src_elt == 0)
	  continue;

	/* STRICT_LOW_PART isn't part of the value BEING set,
	   and neither is the SUBREG inside it.
	   Note that in this case SETS[I].SRC_ELT is really SRC_EQV_ELT.  */
	if (GET_CODE (dest) == STRICT_LOW_PART)
	  dest = SUBREG_REG (XEXP (dest, 0));

	if (REG_P (dest) || GET_CODE (dest) == SUBREG)
	  /* Registers must also be inserted into chains for quantities.  */
	  if (insert_regs (dest, sets[i].src_elt, 1))
	    {
	      /* If `insert_regs' changes something, the hash code must be
		 recalculated.  */
	      rehash_using_reg (dest);
	      sets[i].dest_hash = HASH (dest, GET_MODE (dest));
	    }

	/* If DEST is a paradoxical SUBREG, don't record DEST since the bits
	   outside the mode of GET_MODE (SUBREG_REG (dest)) are undefined.  */
	if (paradoxical_subreg_p (dest))
	  continue;

	elt = insert (dest, sets[i].src_elt,
		      sets[i].dest_hash, GET_MODE (dest));

	/* If this is a constant, insert the constant anchors with the
	   equivalent register-offset expressions using register DEST.  */
	if (targetm.const_anchor
	    && REG_P (dest)
	    && SCALAR_INT_MODE_P (GET_MODE (dest))
	    && GET_CODE (sets[i].src_elt->exp) == CONST_INT)
	  insert_const_anchors (dest, sets[i].src_elt->exp, GET_MODE (dest));

	elt->in_memory = (MEM_P (sets[i].inner_dest)
			  && !MEM_READONLY_P (sets[i].inner_dest));

	/* If we have (set (subreg:m1 (reg:m2 foo) 0) (bar:m1)), M1 is no
	   narrower than M2, and both M1 and M2 are the same number of words,
	   we are also doing (set (reg:m2 foo) (subreg:m2 (bar:m1) 0)) so
	   make that equivalence as well.

	   However, BAR may have equivalences for which gen_lowpart
	   will produce a simpler value than gen_lowpart applied to
	   BAR (e.g., if BAR was ZERO_EXTENDed from M2), so we will scan all
	   BAR's equivalences.  If we don't get a simplified form, make
	   the SUBREG.  It will not be used in an equivalence, but will
	   cause two similar assignments to be detected.

	   Note the loop below will find SUBREG_REG (DEST) since we have
	   already entered SRC and DEST of the SET in the table.  */

	if (GET_CODE (dest) == SUBREG
	    && (known_equal_after_align_down
		(GET_MODE_SIZE (GET_MODE (SUBREG_REG (dest))) - 1,
		 GET_MODE_SIZE (GET_MODE (dest)) - 1,
		 UNITS_PER_WORD))
	    && !partial_subreg_p (dest)
	    && sets[i].src_elt != 0)
	  {
	    machine_mode new_mode = GET_MODE (SUBREG_REG (dest));
	    struct table_elt *elt, *classp = 0;

	    for (elt = sets[i].src_elt->first_same_value; elt;
		 elt = elt->next_same_value)
	      {
		rtx new_src = 0;
		unsigned src_hash;
		struct table_elt *src_elt;

		/* Ignore invalid entries.  */
		if (!REG_P (elt->exp)
		    && ! exp_equiv_p (elt->exp, elt->exp, 1, false))
		  continue;

		/* We may have already been playing subreg games.  If the
		   mode is already correct for the destination, use it.  */
		if (GET_MODE (elt->exp) == new_mode)
		  new_src = elt->exp;
		else
		  {
		    poly_uint64 byte
		      = subreg_lowpart_offset (new_mode, GET_MODE (dest));
		    new_src = simplify_gen_subreg (new_mode, elt->exp,
					           GET_MODE (dest), byte);
		  }

		/* The call to simplify_gen_subreg fails if the value
		   is VOIDmode, yet we can't do any simplification, e.g.
		   for EXPR_LISTs denoting function call results.
		   It is invalid to construct a SUBREG with a VOIDmode
		   SUBREG_REG, hence a zero new_src means we can't do
		   this substitution.  */
		if (! new_src)
		  continue;

		src_hash = HASH (new_src, new_mode);
		src_elt = lookup (new_src, src_hash, new_mode);

		/* Put the new source in the hash table is if isn't
		   already.  */
		if (src_elt == 0)
		  {
		    if (insert_regs (new_src, classp, 0))
		      {
			rehash_using_reg (new_src);
			src_hash = HASH (new_src, new_mode);
		      }
		    src_elt = insert (new_src, classp, src_hash, new_mode);
		    src_elt->in_memory = elt->in_memory;
		    if (GET_CODE (new_src) == ASM_OPERANDS
			&& elt->cost == MAX_COST)
		      src_elt->cost = MAX_COST;
		  }
		else if (classp && classp != src_elt->first_same_value)
		  /* Show that two things that we've seen before are
		     actually the same.  */
		  merge_equiv_classes (src_elt, classp);

		classp = src_elt->first_same_value;
		/* Ignore invalid entries.  */
		while (classp
		       && !REG_P (classp->exp)
		       && ! exp_equiv_p (classp->exp, classp->exp, 1, false))
		  classp = classp->next_same_value;
	      }
	  }
      }

  /* Special handling for (set REG0 REG1) where REG0 is the
     "cheapest", cheaper than REG1.  After cse, REG1 will probably not
     be used in the sequel, so (if easily done) change this insn to
     (set REG1 REG0) and replace REG1 with REG0 in the previous insn
     that computed their value.  Then REG1 will become a dead store
     and won't cloud the situation for later optimizations.

     Do not make this change if REG1 is a hard register, because it will
     then be used in the sequel and we may be changing a two-operand insn
     into a three-operand insn.

     Also do not do this if we are operating on a copy of INSN.  */

  if (n_sets == 1 && sets[0].rtl)
    try_back_substitute_reg (sets[0].rtl, insn);

done:;
}

/* Remove from the hash table all expressions that reference memory.  */

static void
invalidate_memory (void)
{
  int i;
  struct table_elt *p, *next;

  for (i = 0; i < HASH_SIZE; i++)
    for (p = table[i]; p; p = next)
      {
	next = p->next_same_hash;
	if (p->in_memory)
	  remove_from_table (p, i);
      }
}

/* Perform invalidation on the basis of everything about INSN,
   except for invalidating the actual places that are SET in it.
   This includes the places CLOBBERed, and anything that might
   alias with something that is SET or CLOBBERed.  */

static void
invalidate_from_clobbers (rtx_insn *insn)
{
  rtx x = PATTERN (insn);

  if (GET_CODE (x) == CLOBBER)
    {
      rtx ref = XEXP (x, 0);
      if (ref)
	{
	  if (REG_P (ref) || GET_CODE (ref) == SUBREG
	      || MEM_P (ref))
	    invalidate (ref, VOIDmode);
	  else if (GET_CODE (ref) == STRICT_LOW_PART
		   || GET_CODE (ref) == ZERO_EXTRACT)
	    invalidate (XEXP (ref, 0), GET_MODE (ref));
	}
    }
  else if (GET_CODE (x) == PARALLEL)
    {
      int i;
      for (i = XVECLEN (x, 0) - 1; i >= 0; i--)
	{
	  rtx y = XVECEXP (x, 0, i);
	  if (GET_CODE (y) == CLOBBER)
	    {
	      rtx ref = XEXP (y, 0);
	      if (REG_P (ref) || GET_CODE (ref) == SUBREG
		  || MEM_P (ref))
		invalidate (ref, VOIDmode);
	      else if (GET_CODE (ref) == STRICT_LOW_PART
		       || GET_CODE (ref) == ZERO_EXTRACT)
		invalidate (XEXP (ref, 0), GET_MODE (ref));
	    }
	}
    }
}

/* Perform invalidation on the basis of everything about INSN.
   This includes the places CLOBBERed, and anything that might
   alias with something that is SET or CLOBBERed.  */

static void
invalidate_from_sets_and_clobbers (rtx_insn *insn)
{
  rtx tem;
  rtx x = PATTERN (insn);

  if (CALL_P (insn))
    {
      for (tem = CALL_INSN_FUNCTION_USAGE (insn); tem; tem = XEXP (tem, 1))
	{
	  rtx temx = XEXP (tem, 0);
	  if (GET_CODE (temx) == CLOBBER)
	    invalidate (SET_DEST (temx), VOIDmode);
	}
    }

  /* Ensure we invalidate the destination register of a CALL insn.
     This is necessary for machines where this register is a fixed_reg,
     because no other code would invalidate it.  */
  if (GET_CODE (x) == SET && GET_CODE (SET_SRC (x)) == CALL)
    invalidate (SET_DEST (x), VOIDmode);

  else if (GET_CODE (x) == PARALLEL)
    {
      int i;

      for (i = XVECLEN (x, 0) - 1; i >= 0; i--)
	{
	  rtx y = XVECEXP (x, 0, i);
	  if (GET_CODE (y) == CLOBBER)
	    {
	      rtx clobbered = XEXP (y, 0);

	      if (REG_P (clobbered)
		  || GET_CODE (clobbered) == SUBREG)
		invalidate (clobbered, VOIDmode);
	      else if (GET_CODE (clobbered) == STRICT_LOW_PART
		       || GET_CODE (clobbered) == ZERO_EXTRACT)
		invalidate (XEXP (clobbered, 0), GET_MODE (clobbered));
	    }
	  else if (GET_CODE (y) == SET && GET_CODE (SET_SRC (y)) == CALL)
	    invalidate (SET_DEST (y), VOIDmode);
	}
    }
}

static rtx cse_process_note (rtx);

/* A simplify_replace_fn_rtx callback for cse_process_note.  Process X,
   part of the REG_NOTES of an insn.  Replace any registers with either
   an equivalent constant or the canonical form of the register.
   Only replace addresses if the containing MEM remains valid.

   Return the replacement for X, or null if it should be simplified
   recursively.  */

static rtx
cse_process_note_1 (rtx x, const_rtx, void *)
{
  if (MEM_P (x))
    {
      validate_change (x, &XEXP (x, 0), cse_process_note (XEXP (x, 0)), false);
      return x;
    }

  if (REG_P (x))
    {
      int i = REG_QTY (REGNO (x));

      /* Return a constant or a constant register.  */
      if (REGNO_QTY_VALID_P (REGNO (x)))
	{
	  struct qty_table_elem *ent = &qty_table[i];

	  if (ent->const_rtx != NULL_RTX
	      && (CONSTANT_P (ent->const_rtx)
		  || REG_P (ent->const_rtx)))
	    {
	      rtx new_rtx = gen_lowpart (GET_MODE (x), ent->const_rtx);
	      if (new_rtx)
		return copy_rtx (new_rtx);
	    }
	}

      /* Otherwise, canonicalize this register.  */
      return canon_reg (x, NULL);
    }

  return NULL_RTX;
}

/* Process X, part of the REG_NOTES of an insn.  Replace any registers in it
   with either an equivalent constant or the canonical form of the register.
   Only replace addresses if the containing MEM remains valid.  */

static rtx
cse_process_note (rtx x)
{
  return simplify_replace_fn_rtx (x, NULL_RTX, cse_process_note_1, NULL);
}


/* Find a path in the CFG, starting with FIRST_BB to perform CSE on.

   DATA is a pointer to a struct cse_basic_block_data, that is used to
   describe the path.
   It is filled with a queue of basic blocks, starting with FIRST_BB
   and following a trace through the CFG.

   If all paths starting at FIRST_BB have been followed, or no new path
   starting at FIRST_BB can be constructed, this function returns FALSE.
   Otherwise, DATA->path is filled and the function returns TRUE indicating
   that a path to follow was found.

   If FOLLOW_JUMPS is false, the maximum path length is 1 and the only
   block in the path will be FIRST_BB.  */

static bool
cse_find_path (basic_block first_bb, struct cse_basic_block_data *data,
	       int follow_jumps)
{
  basic_block bb;
  edge e;
  int path_size;

  bitmap_set_bit (cse_visited_basic_blocks, first_bb->index);

  /* See if there is a previous path.  */
  path_size = data->path_size;

  /* There is a previous path.  Make sure it started with FIRST_BB.  */
  if (path_size)
    gcc_assert (data->path[0].bb == first_bb);

  /* There was only one basic block in the last path.  Clear the path and
     return, so that paths starting at another basic block can be tried.  */
  if (path_size == 1)
    {
      path_size = 0;
      goto done;
    }

  /* If the path was empty from the beginning, construct a new path.  */
  if (path_size == 0)
    data->path[path_size++].bb = first_bb;
  else
    {
      /* Otherwise, path_size must be equal to or greater than 2, because
	 a previous path exists that is at least two basic blocks long.

	 Update the previous branch path, if any.  If the last branch was
	 previously along the branch edge, take the fallthrough edge now.  */
      while (path_size >= 2)
	{
	  basic_block last_bb_in_path, previous_bb_in_path;
	  edge e;

	  --path_size;
	  last_bb_in_path = data->path[path_size].bb;
	  previous_bb_in_path = data->path[path_size - 1].bb;

	  /* If we previously followed a path along the branch edge, try
	     the fallthru edge now.  */
	  if (EDGE_COUNT (previous_bb_in_path->succs) == 2
	      && any_condjump_p (BB_END (previous_bb_in_path))
	      && (e = find_edge (previous_bb_in_path, last_bb_in_path))
	      && e == BRANCH_EDGE (previous_bb_in_path))
	    {
	      bb = FALLTHRU_EDGE (previous_bb_in_path)->dest;
	      if (bb != EXIT_BLOCK_PTR_FOR_FN (cfun)
		  && single_pred_p (bb)
		  /* We used to assert here that we would only see blocks
		     that we have not visited yet.  But we may end up
		     visiting basic blocks twice if the CFG has changed
		     in this run of cse_main, because when the CFG changes
		     the topological sort of the CFG also changes.  A basic
		     blocks that previously had more than two predecessors
		     may now have a single predecessor, and become part of
		     a path that starts at another basic block.

		     We still want to visit each basic block only once, so
		     halt the path here if we have already visited BB.  */
		  && !bitmap_bit_p (cse_visited_basic_blocks, bb->index))
		{
		  bitmap_set_bit (cse_visited_basic_blocks, bb->index);
		  data->path[path_size++].bb = bb;
		  break;
		}
	    }

	  data->path[path_size].bb = NULL;
	}

      /* If only one block remains in the path, bail.  */
      if (path_size == 1)
	{
	  path_size = 0;
	  goto done;
	}
    }

  /* Extend the path if possible.  */
  if (follow_jumps)
    {
      bb = data->path[path_size - 1].bb;
      while (bb && path_size < param_max_cse_path_length)
	{
	  if (single_succ_p (bb))
	    e = single_succ_edge (bb);
	  else if (EDGE_COUNT (bb->succs) == 2
		   && any_condjump_p (BB_END (bb)))
	    {
	      /* First try to follow the branch.  If that doesn't lead
		 to a useful path, follow the fallthru edge.  */
	      e = BRANCH_EDGE (bb);
	      if (!single_pred_p (e->dest))
		e = FALLTHRU_EDGE (bb);
	    }
	  else
	    e = NULL;

	  if (e
	      && !((e->flags & EDGE_ABNORMAL_CALL) && cfun->has_nonlocal_label)
	      && e->dest != EXIT_BLOCK_PTR_FOR_FN (cfun)
	      && single_pred_p (e->dest)
	      /* Avoid visiting basic blocks twice.  The large comment
		 above explains why this can happen.  */
	      && !bitmap_bit_p (cse_visited_basic_blocks, e->dest->index))
	    {
	      basic_block bb2 = e->dest;
	      bitmap_set_bit (cse_visited_basic_blocks, bb2->index);
	      data->path[path_size++].bb = bb2;
	      bb = bb2;
	    }
	  else
	    bb = NULL;
	}
    }

done:
  data->path_size = path_size;
  return path_size != 0;
}

/* Dump the path in DATA to file F.  NSETS is the number of sets
   in the path.  */

static void
cse_dump_path (struct cse_basic_block_data *data, int nsets, FILE *f)
{
  int path_entry;

  fprintf (f, ";; Following path with %d sets: ", nsets);
  for (path_entry = 0; path_entry < data->path_size; path_entry++)
    fprintf (f, "%d ", (data->path[path_entry].bb)->index);
  fputc ('\n', f);
  fflush (f);
}


/* Return true if BB has exception handling successor edges.  */

static bool
have_eh_succ_edges (basic_block bb)
{
  edge e;
  edge_iterator ei;

  FOR_EACH_EDGE (e, ei, bb->succs)
    if (e->flags & EDGE_EH)
      return true;

  return false;
}


/* Scan to the end of the path described by DATA.  Return an estimate of
   the total number of SETs of all insns in the path.  */

static void
cse_prescan_path (struct cse_basic_block_data *data)
{
  int nsets = 0;
  int path_size = data->path_size;
  int path_entry;

  /* Scan to end of each basic block in the path.  */
  for (path_entry = 0; path_entry < path_size; path_entry++)
    {
      basic_block bb;
      rtx_insn *insn;

      bb = data->path[path_entry].bb;

      FOR_BB_INSNS (bb, insn)
	{
	  if (!INSN_P (insn))
	    continue;

	  /* A PARALLEL can have lots of SETs in it,
	     especially if it is really an ASM_OPERANDS.  */
	  if (GET_CODE (PATTERN (insn)) == PARALLEL)
	    nsets += XVECLEN (PATTERN (insn), 0);
	  else
	    nsets += 1;
	}
    }

  data->nsets = nsets;
}

/* Return true if the pattern of INSN uses a LABEL_REF for which
   there isn't a REG_LABEL_OPERAND note.  */

static bool
check_for_label_ref (rtx_insn *insn)
{
  /* If this insn uses a LABEL_REF and there isn't a REG_LABEL_OPERAND
     note for it, we must rerun jump since it needs to place the note.  If
     this is a LABEL_REF for a CODE_LABEL that isn't in the insn chain,
     don't do this since no REG_LABEL_OPERAND will be added.  */
  subrtx_iterator::array_type array;
  FOR_EACH_SUBRTX (iter, array, PATTERN (insn), ALL)
    {
      const_rtx x = *iter;
      if (GET_CODE (x) == LABEL_REF
	  && !LABEL_REF_NONLOCAL_P (x)
	  && (!JUMP_P (insn)
	      || !label_is_jump_target_p (label_ref_label (x), insn))
	  && LABEL_P (label_ref_label (x))
	  && INSN_UID (label_ref_label (x)) != 0
	  && !find_reg_note (insn, REG_LABEL_OPERAND, label_ref_label (x)))
	return true;
    }
  return false;
}

/* Process a single extended basic block described by EBB_DATA.  */

static void
cse_extended_basic_block (struct cse_basic_block_data *ebb_data)
{
  int path_size = ebb_data->path_size;
  int path_entry;
  int num_insns = 0;

  /* Allocate the space needed by qty_table.  */
  qty_table = XNEWVEC (struct qty_table_elem, max_qty);

  new_basic_block ();
  cse_ebb_live_in = df_get_live_in (ebb_data->path[0].bb);
  cse_ebb_live_out = df_get_live_out (ebb_data->path[path_size - 1].bb);
  for (path_entry = 0; path_entry < path_size; path_entry++)
    {
      basic_block bb;
      rtx_insn *insn;

      bb = ebb_data->path[path_entry].bb;

      /* Invalidate recorded information for eh regs if there is an EH
	 edge pointing to that bb.  */
      if (bb_has_eh_pred (bb))
	{
	  df_ref def;

	  FOR_EACH_ARTIFICIAL_DEF (def, bb->index)
	    if (DF_REF_FLAGS (def) & DF_REF_AT_TOP)
	      invalidate (DF_REF_REG (def), GET_MODE (DF_REF_REG (def)));
	}

      optimize_this_for_speed_p = optimize_bb_for_speed_p (bb);
      FOR_BB_INSNS (bb, insn)
	{
	  /* If we have processed 1,000 insns, flush the hash table to
	     avoid extreme quadratic behavior.  We must not include NOTEs
	     in the count since there may be more of them when generating
	     debugging information.  If we clear the table at different
	     times, code generated with -g -O might be different than code
	     generated with -O but not -g.

	     FIXME: This is a real kludge and needs to be done some other
		    way.  */
	  if (NONDEBUG_INSN_P (insn)
	      && num_insns++ > param_max_cse_insns)
	    {
	      flush_hash_table ();
	      num_insns = 0;
	    }

	  if (INSN_P (insn))
	    {
	      /* Process notes first so we have all notes in canonical forms
		 when looking for duplicate operations.  */
	      bool changed = false;
	      for (rtx note = REG_NOTES (insn); note; note = XEXP (note, 1))
		if (REG_NOTE_KIND (note) == REG_EQUAL)
		  {
		    rtx newval = cse_process_note (XEXP (note, 0));
		    if (newval != XEXP (note, 0))
		      {
			XEXP (note, 0) = newval;
			changed = true;
		      }
		  }
	      if (changed)
		df_notes_rescan (insn);

	      cse_insn (insn);

	      /* If we haven't already found an insn where we added a LABEL_REF,
		 check this one.  */
	      if (INSN_P (insn) && !recorded_label_ref
		  && check_for_label_ref (insn))
		recorded_label_ref = true;
	    }
	}

      /* With non-call exceptions, we are not always able to update
	 the CFG properly inside cse_insn.  So clean up possibly
	 redundant EH edges here.  */
      if (cfun->can_throw_non_call_exceptions && have_eh_succ_edges (bb))
	cse_cfg_altered |= purge_dead_edges (bb);

      /* If we changed a conditional jump, we may have terminated
	 the path we are following.  Check that by verifying that
	 the edge we would take still exists.  If the edge does
	 not exist anymore, purge the remainder of the path.
	 Note that this will cause us to return to the caller.  */
      if (path_entry < path_size - 1)
	{
	  basic_block next_bb = ebb_data->path[path_entry + 1].bb;
	  if (!find_edge (bb, next_bb))
	    {
	      do
		{
		  path_size--;

		  /* If we truncate the path, we must also reset the
		     visited bit on the remaining blocks in the path,
		     or we will never visit them at all.  */
		  bitmap_clear_bit (cse_visited_basic_blocks,
			     ebb_data->path[path_size].bb->index);
		  ebb_data->path[path_size].bb = NULL;
		}
	      while (path_size - 1 != path_entry);
	      ebb_data->path_size = path_size;
	    }
	}

      /* If this is a conditional jump insn, record any known
	 equivalences due to the condition being tested.  */
      insn = BB_END (bb);
      if (path_entry < path_size - 1
	  && EDGE_COUNT (bb->succs) == 2
	  && JUMP_P (insn)
	  && single_set (insn)
	  && any_condjump_p (insn))
	{
	  basic_block next_bb = ebb_data->path[path_entry + 1].bb;
	  bool taken = (next_bb == BRANCH_EDGE (bb)->dest);
	  record_jump_equiv (insn, taken);
	}
    }

  gcc_assert (next_qty <= max_qty);

  free (qty_table);
}


/* Perform cse on the instructions of a function.
   F is the first instruction.
   NREGS is one plus the highest pseudo-reg number used in the instruction.

   Return 2 if jump optimizations should be redone due to simplifications
   in conditional jump instructions.
   Return 1 if the CFG should be cleaned up because it has been modified.
   Return 0 otherwise.  */

static int
cse_main (rtx_insn *f ATTRIBUTE_UNUSED, int nregs)
{
  struct cse_basic_block_data ebb_data;
  basic_block bb;
  int *rc_order = XNEWVEC (int, last_basic_block_for_fn (cfun));
  int i, n_blocks;

  /* CSE doesn't use dominane info but can invalidate it in different ways.
     For simplicity free dominance info here.  */
  free_dominance_info (CDI_DOMINATORS);

  df_set_flags (DF_LR_RUN_DCE);
  df_note_add_problem ();
  df_analyze ();
  df_set_flags (DF_DEFER_INSN_RESCAN);

  reg_scan (get_insns (), max_reg_num ());
  init_cse_reg_info (nregs);

  ebb_data.path = XNEWVEC (struct branch_path,
			   param_max_cse_path_length);

  cse_cfg_altered = false;
  cse_jumps_altered = false;
  recorded_label_ref = false;
  ebb_data.path_size = 0;
  ebb_data.nsets = 0;
  rtl_hooks = cse_rtl_hooks;

  init_recog ();
  init_alias_analysis ();

  reg_eqv_table = XNEWVEC (struct reg_eqv_elem, nregs);

  /* Set up the table of already visited basic blocks.  */
  cse_visited_basic_blocks = sbitmap_alloc (last_basic_block_for_fn (cfun));
  bitmap_clear (cse_visited_basic_blocks);

  /* Loop over basic blocks in reverse completion order (RPO),
     excluding the ENTRY and EXIT blocks.  */
  n_blocks = pre_and_rev_post_order_compute (NULL, rc_order, false);
  i = 0;
  while (i < n_blocks)
    {
      /* Find the first block in the RPO queue that we have not yet
	 processed before.  */
      do
	{
	  bb = BASIC_BLOCK_FOR_FN (cfun, rc_order[i++]);
	}
      while (bitmap_bit_p (cse_visited_basic_blocks, bb->index)
	     && i < n_blocks);

      /* Find all paths starting with BB, and process them.  */
      while (cse_find_path (bb, &ebb_data, flag_cse_follow_jumps))
	{
	  /* Pre-scan the path.  */
	  cse_prescan_path (&ebb_data);

	  /* If this basic block has no sets, skip it.  */
	  if (ebb_data.nsets == 0)
	    continue;

	  /* Get a reasonable estimate for the maximum number of qty's
	     needed for this path.  For this, we take the number of sets
	     and multiply that by MAX_RECOG_OPERANDS.  */
	  max_qty = ebb_data.nsets * MAX_RECOG_OPERANDS;

	  /* Dump the path we're about to process.  */
	  if (dump_file)
	    cse_dump_path (&ebb_data, ebb_data.nsets, dump_file);

	  cse_extended_basic_block (&ebb_data);
	}
    }

  /* Clean up.  */
  end_alias_analysis ();
  free (reg_eqv_table);
  free (ebb_data.path);
  sbitmap_free (cse_visited_basic_blocks);
  free (rc_order);
  rtl_hooks = general_rtl_hooks;

  if (cse_jumps_altered || recorded_label_ref)
    return 2;
  else if (cse_cfg_altered)
    return 1;
  else
    return 0;
}

/* Count the number of times registers are used (not set) in X.
   COUNTS is an array in which we accumulate the count, INCR is how much
   we count each register usage.

   Don't count a usage of DEST, which is the SET_DEST of a SET which
   contains X in its SET_SRC.  This is because such a SET does not
   modify the liveness of DEST.
   DEST is set to pc_rtx for a trapping insn, or for an insn with side effects.
   We must then count uses of a SET_DEST regardless, because the insn can't be
   deleted here.  */

static void
count_reg_usage (rtx x, int *counts, rtx dest, int incr)
{
  enum rtx_code code;
  rtx note;
  const char *fmt;
  int i, j;

  if (x == 0)
    return;

  switch (code = GET_CODE (x))
    {
    case REG:
      if (x != dest)
	counts[REGNO (x)] += incr;
      return;

    case PC:
    case CONST:
    CASE_CONST_ANY:
    case SYMBOL_REF:
    case LABEL_REF:
      return;

    case CLOBBER:
      /* If we are clobbering a MEM, mark any registers inside the address
         as being used.  */
      if (MEM_P (XEXP (x, 0)))
	count_reg_usage (XEXP (XEXP (x, 0), 0), counts, NULL_RTX, incr);
      return;

    case SET:
      /* Unless we are setting a REG, count everything in SET_DEST.  */
      if (!REG_P (SET_DEST (x)))
	count_reg_usage (SET_DEST (x), counts, NULL_RTX, incr);
      count_reg_usage (SET_SRC (x), counts,
		       dest ? dest : SET_DEST (x),
		       incr);
      return;

    case DEBUG_INSN:
      return;

    case CALL_INSN:
    case INSN:
    case JUMP_INSN:
      /* We expect dest to be NULL_RTX here.  If the insn may throw,
	 or if it cannot be deleted due to side-effects, mark this fact
	 by setting DEST to pc_rtx.  */
      if ((!cfun->can_delete_dead_exceptions && !insn_nothrow_p (x))
	  || side_effects_p (PATTERN (x)))
	dest = pc_rtx;
      if (code == CALL_INSN)
	count_reg_usage (CALL_INSN_FUNCTION_USAGE (x), counts, dest, incr);
      count_reg_usage (PATTERN (x), counts, dest, incr);

      /* Things used in a REG_EQUAL note aren't dead since loop may try to
	 use them.  */

      note = find_reg_equal_equiv_note (x);
      if (note)
	{
	  rtx eqv = XEXP (note, 0);

	  if (GET_CODE (eqv) == EXPR_LIST)
	  /* This REG_EQUAL note describes the result of a function call.
	     Process all the arguments.  */
	    do
	      {
		count_reg_usage (XEXP (eqv, 0), counts, dest, incr);
		eqv = XEXP (eqv, 1);
	      }
	    while (eqv && GET_CODE (eqv) == EXPR_LIST);
	  else
	    count_reg_usage (eqv, counts, dest, incr);
	}
      return;

    case EXPR_LIST:
      if (REG_NOTE_KIND (x) == REG_EQUAL
	  || (REG_NOTE_KIND (x) != REG_NONNEG && GET_CODE (XEXP (x,0)) == USE)
	  /* FUNCTION_USAGE expression lists may include (CLOBBER (mem /u)),
	     involving registers in the address.  */
	  || GET_CODE (XEXP (x, 0)) == CLOBBER)
	count_reg_usage (XEXP (x, 0), counts, NULL_RTX, incr);

      count_reg_usage (XEXP (x, 1), counts, NULL_RTX, incr);
      return;

    case ASM_OPERANDS:
      /* Iterate over just the inputs, not the constraints as well.  */
      for (i = ASM_OPERANDS_INPUT_LENGTH (x) - 1; i >= 0; i--)
	count_reg_usage (ASM_OPERANDS_INPUT (x, i), counts, dest, incr);
      return;

    case INSN_LIST:
    case INT_LIST:
      gcc_unreachable ();

    default:
      break;
    }

  fmt = GET_RTX_FORMAT (code);
  for (i = GET_RTX_LENGTH (code) - 1; i >= 0; i--)
    {
      if (fmt[i] == 'e')
	count_reg_usage (XEXP (x, i), counts, dest, incr);
      else if (fmt[i] == 'E')
	for (j = XVECLEN (x, i) - 1; j >= 0; j--)
	  count_reg_usage (XVECEXP (x, i, j), counts, dest, incr);
    }
}

/* Return true if X is a dead register.  */

static inline int
is_dead_reg (const_rtx x, int *counts)
{
  return (REG_P (x)
	  && REGNO (x) >= FIRST_PSEUDO_REGISTER
	  && counts[REGNO (x)] == 0);
}

/* Return true if set is live.  */
static bool
set_live_p (rtx set, int *counts)
{
  if (set_noop_p (set))
    return false;

  if (!is_dead_reg (SET_DEST (set), counts)
      || side_effects_p (SET_SRC (set)))
    return true;

  return false;
}

/* Return true if insn is live.  */

static bool
insn_live_p (rtx_insn *insn, int *counts)
{
  int i;
  if (!cfun->can_delete_dead_exceptions && !insn_nothrow_p (insn))
    return true;
  else if (GET_CODE (PATTERN (insn)) == SET)
    return set_live_p (PATTERN (insn), counts);
  else if (GET_CODE (PATTERN (insn)) == PARALLEL)
    {
      for (i = XVECLEN (PATTERN (insn), 0) - 1; i >= 0; i--)
	{
	  rtx elt = XVECEXP (PATTERN (insn), 0, i);

	  if (GET_CODE (elt) == SET)
	    {
	      if (set_live_p (elt, counts))
		return true;
	    }
	  else if (GET_CODE (elt) != CLOBBER && GET_CODE (elt) != USE)
	    return true;
	}
      return false;
    }
  else if (DEBUG_INSN_P (insn))
    {
      rtx_insn *next;

      if (DEBUG_MARKER_INSN_P (insn))
	return true;

      for (next = NEXT_INSN (insn); next; next = NEXT_INSN (next))
	if (NOTE_P (next))
	  continue;
	else if (!DEBUG_INSN_P (next))
	  return true;
	/* If we find an inspection point, such as a debug begin stmt,
	   we want to keep the earlier debug insn.  */
	else if (DEBUG_MARKER_INSN_P (next))
	  return true;
	else if (INSN_VAR_LOCATION_DECL (insn) == INSN_VAR_LOCATION_DECL (next))
	  return false;

      return true;
    }
  else
    return true;
}

/* Count the number of stores into pseudo.  Callback for note_stores.  */

static void
count_stores (rtx x, const_rtx set ATTRIBUTE_UNUSED, void *data)
{
  int *counts = (int *) data;
  if (REG_P (x) && REGNO (x) >= FIRST_PSEUDO_REGISTER)
    counts[REGNO (x)]++;
}

/* Return if DEBUG_INSN pattern PAT needs to be reset because some dead
   pseudo doesn't have a replacement.  COUNTS[X] is zero if register X
   is dead and REPLACEMENTS[X] is null if it has no replacemenet.
   Set *SEEN_REPL to true if we see a dead register that does have
   a replacement.  */

static bool
is_dead_debug_insn (const_rtx pat, int *counts, rtx *replacements,
		    bool *seen_repl)
{
  subrtx_iterator::array_type array;
  FOR_EACH_SUBRTX (iter, array, pat, NONCONST)
    {
      const_rtx x = *iter;
      if (is_dead_reg (x, counts))
	{
	  if (replacements && replacements[REGNO (x)] != NULL_RTX)
	    *seen_repl = true;
	  else
	    return true;
	}
    }
  return false;
}

/* Replace a dead pseudo in a DEBUG_INSN with replacement DEBUG_EXPR.
   Callback for simplify_replace_fn_rtx.  */

static rtx
replace_dead_reg (rtx x, const_rtx old_rtx ATTRIBUTE_UNUSED, void *data)
{
  rtx *replacements = (rtx *) data;

  if (REG_P (x)
      && REGNO (x) >= FIRST_PSEUDO_REGISTER
      && replacements[REGNO (x)] != NULL_RTX)
    {
      if (GET_MODE (x) == GET_MODE (replacements[REGNO (x)]))
	return replacements[REGNO (x)];
      return lowpart_subreg (GET_MODE (x), replacements[REGNO (x)],
			     GET_MODE (replacements[REGNO (x)]));
    }
  return NULL_RTX;
}

/* Scan all the insns and delete any that are dead; i.e., they store a register
   that is never used or they copy a register to itself.

   This is used to remove insns made obviously dead by cse, loop or other
   optimizations.  It improves the heuristics in loop since it won't try to
   move dead invariants out of loops or make givs for dead quantities.  The
   remaining passes of the compilation are also sped up.  */

int
delete_trivially_dead_insns (rtx_insn *insns, int nreg)
{
  int *counts;
  rtx_insn *insn, *prev;
  rtx *replacements = NULL;
  int ndead = 0;

  timevar_push (TV_DELETE_TRIVIALLY_DEAD);
  /* First count the number of times each register is used.  */
  if (MAY_HAVE_DEBUG_BIND_INSNS)
    {
      counts = XCNEWVEC (int, nreg * 3);
      for (insn = insns; insn; insn = NEXT_INSN (insn))
	if (DEBUG_BIND_INSN_P (insn))
	  count_reg_usage (INSN_VAR_LOCATION_LOC (insn), counts + nreg,
			   NULL_RTX, 1);
	else if (INSN_P (insn))
	  {
	    count_reg_usage (insn, counts, NULL_RTX, 1);
	    note_stores (insn, count_stores, counts + nreg * 2);
	  }
      /* If there can be debug insns, COUNTS are 3 consecutive arrays.
	 First one counts how many times each pseudo is used outside
	 of debug insns, second counts how many times each pseudo is
	 used in debug insns and third counts how many times a pseudo
	 is stored.  */
    }
  else
    {
      counts = XCNEWVEC (int, nreg);
      for (insn = insns; insn; insn = NEXT_INSN (insn))
	if (INSN_P (insn))
	  count_reg_usage (insn, counts, NULL_RTX, 1);
      /* If no debug insns can be present, COUNTS is just an array
	 which counts how many times each pseudo is used.  */
    }
  /* Pseudo PIC register should be considered as used due to possible
     new usages generated.  */
  if (!reload_completed
      && pic_offset_table_rtx
      && REGNO (pic_offset_table_rtx) >= FIRST_PSEUDO_REGISTER)
    counts[REGNO (pic_offset_table_rtx)]++;
  /* Go from the last insn to the first and delete insns that only set unused
     registers or copy a register to itself.  As we delete an insn, remove
     usage counts for registers it uses.

     The first jump optimization pass may leave a real insn as the last
     insn in the function.   We must not skip that insn or we may end
     up deleting code that is not really dead.

     If some otherwise unused register is only used in DEBUG_INSNs,
     try to create a DEBUG_EXPR temporary and emit a DEBUG_INSN before
     the setter.  Then go through DEBUG_INSNs and if a DEBUG_EXPR
     has been created for the unused register, replace it with
     the DEBUG_EXPR, otherwise reset the DEBUG_INSN.  */
  for (insn = get_last_insn (); insn; insn = prev)
    {
      int live_insn = 0;

      prev = PREV_INSN (insn);
      if (!INSN_P (insn))
	continue;

      live_insn = insn_live_p (insn, counts);

      /* If this is a dead insn, delete it and show registers in it aren't
	 being used.  */

      if (! live_insn && dbg_cnt (delete_trivial_dead))
	{
	  if (DEBUG_INSN_P (insn))
	    {
	      if (DEBUG_BIND_INSN_P (insn))
		count_reg_usage (INSN_VAR_LOCATION_LOC (insn), counts + nreg,
				 NULL_RTX, -1);
	    }
	  else
	    {
	      rtx set;
	      if (MAY_HAVE_DEBUG_BIND_INSNS
		  && (set = single_set (insn)) != NULL_RTX
		  && is_dead_reg (SET_DEST (set), counts)
		  /* Used at least once in some DEBUG_INSN.  */
		  && counts[REGNO (SET_DEST (set)) + nreg] > 0
		  /* And set exactly once.  */
		  && counts[REGNO (SET_DEST (set)) + nreg * 2] == 1
		  && !side_effects_p (SET_SRC (set))
		  && asm_noperands (PATTERN (insn)) < 0)
		{
		  rtx dval, bind_var_loc;
		  rtx_insn *bind;

		  /* Create DEBUG_EXPR (and DEBUG_EXPR_DECL).  */
		  dval = make_debug_expr_from_rtl (SET_DEST (set));

		  /* Emit a debug bind insn before the insn in which
		     reg dies.  */
		  bind_var_loc =
		    gen_rtx_VAR_LOCATION (GET_MODE (SET_DEST (set)),
					  DEBUG_EXPR_TREE_DECL (dval),
					  SET_SRC (set),
					  VAR_INIT_STATUS_INITIALIZED);
		  count_reg_usage (bind_var_loc, counts + nreg, NULL_RTX, 1);

		  bind = emit_debug_insn_before (bind_var_loc, insn);
		  df_insn_rescan (bind);

		  if (replacements == NULL)
		    replacements = XCNEWVEC (rtx, nreg);
		  replacements[REGNO (SET_DEST (set))] = dval;
		}

	      count_reg_usage (insn, counts, NULL_RTX, -1);
	      ndead++;
	    }
	  cse_cfg_altered |= delete_insn_and_edges (insn);
	}
    }

  if (MAY_HAVE_DEBUG_BIND_INSNS)
    {
      for (insn = get_last_insn (); insn; insn = PREV_INSN (insn))
	if (DEBUG_BIND_INSN_P (insn))
	  {
	    /* If this debug insn references a dead register that wasn't replaced
	       with an DEBUG_EXPR, reset the DEBUG_INSN.  */
	    bool seen_repl = false;
	    if (is_dead_debug_insn (INSN_VAR_LOCATION_LOC (insn),
				    counts, replacements, &seen_repl))
	      {
		INSN_VAR_LOCATION_LOC (insn) = gen_rtx_UNKNOWN_VAR_LOC ();
		df_insn_rescan (insn);
	      }
	    else if (seen_repl)
	      {
		INSN_VAR_LOCATION_LOC (insn)
		  = simplify_replace_fn_rtx (INSN_VAR_LOCATION_LOC (insn),
					     NULL_RTX, replace_dead_reg,
					     replacements);
		df_insn_rescan (insn);
	      }
	  }
      free (replacements);
    }

  if (dump_file && ndead)
    fprintf (dump_file, "Deleted %i trivially dead insns\n",
	     ndead);
  /* Clean up.  */
  free (counts);
  timevar_pop (TV_DELETE_TRIVIALLY_DEAD);
  return ndead;
}

/* If LOC contains references to NEWREG in a different mode, change them
   to use NEWREG instead.  */

static void
cse_change_cc_mode (subrtx_ptr_iterator::array_type &array,
		    rtx *loc, rtx_insn *insn, rtx newreg)
{
  FOR_EACH_SUBRTX_PTR (iter, array, loc, NONCONST)
    {
      rtx *loc = *iter;
      rtx x = *loc;
      if (x
	  && REG_P (x)
	  && REGNO (x) == REGNO (newreg)
	  && GET_MODE (x) != GET_MODE (newreg))
	{
	  validate_change (insn, loc, newreg, 1);
	  iter.skip_subrtxes ();
	}
    }
}

/* Change the mode of any reference to the register REGNO (NEWREG) to
   GET_MODE (NEWREG) in INSN.  */

static void
cse_change_cc_mode_insn (rtx_insn *insn, rtx newreg)
{
  int success;

  if (!INSN_P (insn))
    return;

  subrtx_ptr_iterator::array_type array;
  cse_change_cc_mode (array, &PATTERN (insn), insn, newreg);
  cse_change_cc_mode (array, &REG_NOTES (insn), insn, newreg);

  /* If the following assertion was triggered, there is most probably
     something wrong with the cc_modes_compatible back end function.
     CC modes only can be considered compatible if the insn - with the mode
     replaced by any of the compatible modes - can still be recognized.  */
  success = apply_change_group ();
  gcc_assert (success);
}

/* Change the mode of any reference to the register REGNO (NEWREG) to
   GET_MODE (NEWREG), starting at START.  Stop before END.  Stop at
   any instruction which modifies NEWREG.  */

static void
cse_change_cc_mode_insns (rtx_insn *start, rtx_insn *end, rtx newreg)
{
  rtx_insn *insn;

  for (insn = start; insn != end; insn = NEXT_INSN (insn))
    {
      if (! INSN_P (insn))
	continue;

      if (reg_set_p (newreg, insn))
	return;

      cse_change_cc_mode_insn (insn, newreg);
    }
}

/* BB is a basic block which finishes with CC_REG as a condition code
   register which is set to CC_SRC.  Look through the successors of BB
   to find blocks which have a single predecessor (i.e., this one),
   and look through those blocks for an assignment to CC_REG which is
   equivalent to CC_SRC.  CAN_CHANGE_MODE indicates whether we are
   permitted to change the mode of CC_SRC to a compatible mode.  This
   returns VOIDmode if no equivalent assignments were found.
   Otherwise it returns the mode which CC_SRC should wind up with.
   ORIG_BB should be the same as BB in the outermost cse_cc_succs call,
   but is passed unmodified down to recursive calls in order to prevent
   endless recursion.

   The main complexity in this function is handling the mode issues.
   We may have more than one duplicate which we can eliminate, and we
   try to find a mode which will work for multiple duplicates.  */

static machine_mode
cse_cc_succs (basic_block bb, basic_block orig_bb, rtx cc_reg, rtx cc_src,
	      bool can_change_mode)
{
  bool found_equiv;
  machine_mode mode;
  unsigned int insn_count;
  edge e;
  rtx_insn *insns[2];
  machine_mode modes[2];
  rtx_insn *last_insns[2];
  unsigned int i;
  rtx newreg;
  edge_iterator ei;

  /* We expect to have two successors.  Look at both before picking
     the final mode for the comparison.  If we have more successors
     (i.e., some sort of table jump, although that seems unlikely),
     then we require all beyond the first two to use the same
     mode.  */

  found_equiv = false;
  mode = GET_MODE (cc_src);
  insn_count = 0;
  FOR_EACH_EDGE (e, ei, bb->succs)
    {
      rtx_insn *insn;
      rtx_insn *end;

      if (e->flags & EDGE_COMPLEX)
	continue;

      if (EDGE_COUNT (e->dest->preds) != 1
	  || e->dest == EXIT_BLOCK_PTR_FOR_FN (cfun)
	  /* Avoid endless recursion on unreachable blocks.  */
	  || e->dest == orig_bb)
	continue;

      end = NEXT_INSN (BB_END (e->dest));
      for (insn = BB_HEAD (e->dest); insn != end; insn = NEXT_INSN (insn))
	{
	  rtx set;

	  if (! INSN_P (insn))
	    continue;

	  /* If CC_SRC is modified, we have to stop looking for
	     something which uses it.  */
	  if (modified_in_p (cc_src, insn))
	    break;

	  /* Check whether INSN sets CC_REG to CC_SRC.  */
	  set = single_set (insn);
	  if (set
	      && REG_P (SET_DEST (set))
	      && REGNO (SET_DEST (set)) == REGNO (cc_reg))
	    {
	      bool found;
	      machine_mode set_mode;
	      machine_mode comp_mode;

	      found = false;
	      set_mode = GET_MODE (SET_SRC (set));
	      comp_mode = set_mode;
	      if (rtx_equal_p (cc_src, SET_SRC (set)))
		found = true;
	      else if (GET_CODE (cc_src) == COMPARE
		       && GET_CODE (SET_SRC (set)) == COMPARE
		       && mode != set_mode
		       && rtx_equal_p (XEXP (cc_src, 0),
				       XEXP (SET_SRC (set), 0))
		       && rtx_equal_p (XEXP (cc_src, 1),
				       XEXP (SET_SRC (set), 1)))

		{
		  comp_mode = targetm.cc_modes_compatible (mode, set_mode);
		  if (comp_mode != VOIDmode
		      && (can_change_mode || comp_mode == mode))
		    found = true;
		}

	      if (found)
		{
		  found_equiv = true;
		  if (insn_count < ARRAY_SIZE (insns))
		    {
		      insns[insn_count] = insn;
		      modes[insn_count] = set_mode;
		      last_insns[insn_count] = end;
		      ++insn_count;

		      if (mode != comp_mode)
			{
			  gcc_assert (can_change_mode);
			  mode = comp_mode;

			  /* The modified insn will be re-recognized later.  */
			  PUT_MODE (cc_src, mode);
			}
		    }
		  else
		    {
		      if (set_mode != mode)
			{
			  /* We found a matching expression in the
			     wrong mode, but we don't have room to
			     store it in the array.  Punt.  This case
			     should be rare.  */
			  break;
			}
		      /* INSN sets CC_REG to a value equal to CC_SRC
			 with the right mode.  We can simply delete
			 it.  */
		      delete_insn (insn);
		    }

		  /* We found an instruction to delete.  Keep looking,
		     in the hopes of finding a three-way jump.  */
		  continue;
		}

	      /* We found an instruction which sets the condition
		 code, so don't look any farther.  */
	      break;
	    }

	  /* If INSN sets CC_REG in some other way, don't look any
	     farther.  */
	  if (reg_set_p (cc_reg, insn))
	    break;
	}

      /* If we fell off the bottom of the block, we can keep looking
	 through successors.  We pass CAN_CHANGE_MODE as false because
	 we aren't prepared to handle compatibility between the
	 further blocks and this block.  */
      if (insn == end)
	{
	  machine_mode submode;

	  submode = cse_cc_succs (e->dest, orig_bb, cc_reg, cc_src, false);
	  if (submode != VOIDmode)
	    {
	      gcc_assert (submode == mode);
	      found_equiv = true;
	      can_change_mode = false;
	    }
	}
    }

  if (! found_equiv)
    return VOIDmode;

  /* Now INSN_COUNT is the number of instructions we found which set
     CC_REG to a value equivalent to CC_SRC.  The instructions are in
     INSNS.  The modes used by those instructions are in MODES.  */

  newreg = NULL_RTX;
  for (i = 0; i < insn_count; ++i)
    {
      if (modes[i] != mode)
	{
	  /* We need to change the mode of CC_REG in INSNS[i] and
	     subsequent instructions.  */
	  if (! newreg)
	    {
	      if (GET_MODE (cc_reg) == mode)
		newreg = cc_reg;
	      else
		newreg = gen_rtx_REG (mode, REGNO (cc_reg));
	    }
	  cse_change_cc_mode_insns (NEXT_INSN (insns[i]), last_insns[i],
				    newreg);
	}

      cse_cfg_altered |= delete_insn_and_edges (insns[i]);
    }

  return mode;
}

/* If we have a fixed condition code register (or two), walk through
   the instructions and try to eliminate duplicate assignments.  */

static void
cse_condition_code_reg (void)
{
  unsigned int cc_regno_1;
  unsigned int cc_regno_2;
  rtx cc_reg_1;
  rtx cc_reg_2;
  basic_block bb;

  if (! targetm.fixed_condition_code_regs (&cc_regno_1, &cc_regno_2))
    return;

  cc_reg_1 = gen_rtx_REG (CCmode, cc_regno_1);
  if (cc_regno_2 != INVALID_REGNUM)
    cc_reg_2 = gen_rtx_REG (CCmode, cc_regno_2);
  else
    cc_reg_2 = NULL_RTX;

  FOR_EACH_BB_FN (bb, cfun)
    {
      rtx_insn *last_insn;
      rtx cc_reg;
      rtx_insn *insn;
      rtx_insn *cc_src_insn;
      rtx cc_src;
      machine_mode mode;
      machine_mode orig_mode;

      /* Look for blocks which end with a conditional jump based on a
	 condition code register.  Then look for the instruction which
	 sets the condition code register.  Then look through the
	 successor blocks for instructions which set the condition
	 code register to the same value.  There are other possible
	 uses of the condition code register, but these are by far the
	 most common and the ones which we are most likely to be able
	 to optimize.  */

      last_insn = BB_END (bb);
      if (!JUMP_P (last_insn))
	continue;

      if (reg_referenced_p (cc_reg_1, PATTERN (last_insn)))
	cc_reg = cc_reg_1;
      else if (cc_reg_2 && reg_referenced_p (cc_reg_2, PATTERN (last_insn)))
	cc_reg = cc_reg_2;
      else
	continue;

      cc_src_insn = NULL;
      cc_src = NULL_RTX;
      for (insn = PREV_INSN (last_insn);
	   insn && insn != PREV_INSN (BB_HEAD (bb));
	   insn = PREV_INSN (insn))
	{
	  rtx set;

	  if (! INSN_P (insn))
	    continue;
	  set = single_set (insn);
	  if (set
	      && REG_P (SET_DEST (set))
	      && REGNO (SET_DEST (set)) == REGNO (cc_reg))
	    {
	      cc_src_insn = insn;
	      cc_src = SET_SRC (set);
	      break;
	    }
	  else if (reg_set_p (cc_reg, insn))
	    break;
	}

      if (! cc_src_insn)
	continue;

      if (modified_between_p (cc_src, cc_src_insn, NEXT_INSN (last_insn)))
	continue;

      /* Now CC_REG is a condition code register used for a
	 conditional jump at the end of the block, and CC_SRC, in
	 CC_SRC_INSN, is the value to which that condition code
	 register is set, and CC_SRC is still meaningful at the end of
	 the basic block.  */

      orig_mode = GET_MODE (cc_src);
      mode = cse_cc_succs (bb, bb, cc_reg, cc_src, true);
      if (mode != VOIDmode)
	{
	  gcc_assert (mode == GET_MODE (cc_src));
	  if (mode != orig_mode)
	    {
	      rtx newreg = gen_rtx_REG (mode, REGNO (cc_reg));

	      cse_change_cc_mode_insn (cc_src_insn, newreg);

	      /* Do the same in the following insns that use the
		 current value of CC_REG within BB.  */
	      cse_change_cc_mode_insns (NEXT_INSN (cc_src_insn),
					NEXT_INSN (last_insn),
					newreg);
	    }
	}
    }
}


/* Perform common subexpression elimination.  Nonzero value from
   `cse_main' means that jumps were simplified and some code may now
   be unreachable, so do jump optimization again.  */
static unsigned int
rest_of_handle_cse (void)
{
  int tem;

  if (dump_file)
    dump_flow_info (dump_file, dump_flags);

  tem = cse_main (get_insns (), max_reg_num ());

  /* If we are not running more CSE passes, then we are no longer
     expecting CSE to be run.  But always rerun it in a cheap mode.  */
  cse_not_expected = !flag_rerun_cse_after_loop && !flag_gcse;

  if (tem == 2)
    {
      timevar_push (TV_JUMP);
      rebuild_jump_labels (get_insns ());
      cse_cfg_altered |= cleanup_cfg (CLEANUP_CFG_CHANGED);
      timevar_pop (TV_JUMP);
    }
  else if (tem == 1 || optimize > 1)
    cse_cfg_altered |= cleanup_cfg (0);

  return 0;
}

namespace {

const pass_data pass_data_cse =
{
  RTL_PASS, /* type */
  "cse1", /* name */
  OPTGROUP_NONE, /* optinfo_flags */
  TV_CSE, /* tv_id */
  0, /* properties_required */
  0, /* properties_provided */
  0, /* properties_destroyed */
  0, /* todo_flags_start */
  TODO_df_finish, /* todo_flags_finish */
};

class pass_cse : public rtl_opt_pass
{
public:
  pass_cse (gcc::context *ctxt)
    : rtl_opt_pass (pass_data_cse, ctxt)
  {}

  /* opt_pass methods: */
  virtual bool gate (function *) { return optimize > 0; }
  virtual unsigned int execute (function *) { return rest_of_handle_cse (); }

}; // class pass_cse

} // anon namespace

rtl_opt_pass *
make_pass_cse (gcc::context *ctxt)
{
  return new pass_cse (ctxt);
}


/* Run second CSE pass after loop optimizations.  */
static unsigned int
rest_of_handle_cse2 (void)
{
  int tem;

  if (dump_file)
    dump_flow_info (dump_file, dump_flags);

  tem = cse_main (get_insns (), max_reg_num ());

  /* Run a pass to eliminate duplicated assignments to condition code
     registers.  We have to run this after bypass_jumps, because it
     makes it harder for that pass to determine whether a jump can be
     bypassed safely.  */
  cse_condition_code_reg ();

  delete_trivially_dead_insns (get_insns (), max_reg_num ());

  if (tem == 2)
    {
      timevar_push (TV_JUMP);
      rebuild_jump_labels (get_insns ());
      cse_cfg_altered |= cleanup_cfg (CLEANUP_CFG_CHANGED);
      timevar_pop (TV_JUMP);
    }
  else if (tem == 1 || cse_cfg_altered)
    cse_cfg_altered |= cleanup_cfg (0);

  cse_not_expected = 1;
  return 0;
}


namespace {

const pass_data pass_data_cse2 =
{
  RTL_PASS, /* type */
  "cse2", /* name */
  OPTGROUP_NONE, /* optinfo_flags */
  TV_CSE2, /* tv_id */
  0, /* properties_required */
  0, /* properties_provided */
  0, /* properties_destroyed */
  0, /* todo_flags_start */
  TODO_df_finish, /* todo_flags_finish */
};

class pass_cse2 : public rtl_opt_pass
{
public:
  pass_cse2 (gcc::context *ctxt)
    : rtl_opt_pass (pass_data_cse2, ctxt)
  {}

  /* opt_pass methods: */
  virtual bool gate (function *)
    {
      return optimize > 0 && flag_rerun_cse_after_loop;
    }

  virtual unsigned int execute (function *) { return rest_of_handle_cse2 (); }

}; // class pass_cse2

} // anon namespace

rtl_opt_pass *
make_pass_cse2 (gcc::context *ctxt)
{
  return new pass_cse2 (ctxt);
}

/* Run second CSE pass after loop optimizations.  */
static unsigned int
rest_of_handle_cse_after_global_opts (void)
{
  int save_cfj;
  int tem;

  /* We only want to do local CSE, so don't follow jumps.  */
  save_cfj = flag_cse_follow_jumps;
  flag_cse_follow_jumps = 0;

  rebuild_jump_labels (get_insns ());
  tem = cse_main (get_insns (), max_reg_num ());
  cse_cfg_altered |= purge_all_dead_edges ();
  delete_trivially_dead_insns (get_insns (), max_reg_num ());

  cse_not_expected = !flag_rerun_cse_after_loop;

  /* If cse altered any jumps, rerun jump opts to clean things up.  */
  if (tem == 2)
    {
      timevar_push (TV_JUMP);
      rebuild_jump_labels (get_insns ());
      cse_cfg_altered |= cleanup_cfg (CLEANUP_CFG_CHANGED);
      timevar_pop (TV_JUMP);
    }
  else if (tem == 1 || cse_cfg_altered)
    cse_cfg_altered |= cleanup_cfg (0);

  flag_cse_follow_jumps = save_cfj;
  return 0;
}

namespace {

const pass_data pass_data_cse_after_global_opts =
{
  RTL_PASS, /* type */
  "cse_local", /* name */
  OPTGROUP_NONE, /* optinfo_flags */
  TV_CSE, /* tv_id */
  0, /* properties_required */
  0, /* properties_provided */
  0, /* properties_destroyed */
  0, /* todo_flags_start */
  TODO_df_finish, /* todo_flags_finish */
};

class pass_cse_after_global_opts : public rtl_opt_pass
{
public:
  pass_cse_after_global_opts (gcc::context *ctxt)
    : rtl_opt_pass (pass_data_cse_after_global_opts, ctxt)
  {}

  /* opt_pass methods: */
  virtual bool gate (function *)
    {
      return optimize > 0 && flag_rerun_cse_after_global_opts;
    }

  virtual unsigned int execute (function *)
    {
      return rest_of_handle_cse_after_global_opts ();
    }

}; // class pass_cse_after_global_opts

} // anon namespace

rtl_opt_pass *
make_pass_cse_after_global_opts (gcc::context *ctxt)
{
  return new pass_cse_after_global_opts (ctxt);
}