s390x: Add support for atomic operations (part 1)

This adds full back-end support for the Fence, AtomicLoad
and AtomicStore operations, and partial support for the
AtomicCas and AtomicRmw operations.

The missing pieces include sub-word operations, operations
on little-endian memory requiring byte-swapping, and some
of the subtypes of AtomicRmw -- everything that cannot be
implemented without a compare-and-swap loop.  This will be
done in a follow-up patch.

This patch already suffices to make the test suite green
again after a recent change that now requires atomic
operations when accessing the heap.

cranelift/filetests/filetests/isa/s390x/atomic_cas.clif

@@ -0,0 +1,25 @@
+test compile
+target s390x
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; ATOMIC_CAS
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+function %atomic_cas_i64(i64, i64, i64) -> i64 {
+block0(v0: i64, v1: i64, v2: i64):
+  v3 = atomic_cas.i64 v2, v0, v1
+  return v3
+}
+
+; check:  csg %r2, %r3, 0(%r4)
+; nextln: br %r14
+
+function %atomic_cas_i32(i32, i32, i64) -> i32 {
+block0(v0: i32, v1: i32, v2: i64):
+  v3 = atomic_cas.i32 v2, v0, v1
+  return v3
+}
+
+; check:  cs %r2, %r3, 0(%r4)
+; nextln: br %r14
+

cranelift/filetests/filetests/isa/s390x/atomic_load-little.clif

@@ -0,0 +1,72 @@
+test compile
+target s390x
+
+function %atomic_load_i64(i64) -> i64 {
+block0(v0: i64):
+  v1 = atomic_load.i64 little v0
+  return v1
+}
+
+; check:  lrvg %r2, 0(%r2)
+; nextln: br %r14
+
+function %atomic_load_i64_sym() -> i64 {
+  gv0 = symbol colocated %sym
+block0:
+  v0 = symbol_value.i64 gv0
+  v1 = atomic_load.i64 little v0
+  return v1
+}
+
+; check:  larl %r1, %sym + 0 ; lrvg %r2, 0(%r1)
+; nextln: br %r14
+
+function %atomic_load_i32(i64) -> i32 {
+block0(v0: i64):
+  v1 = atomic_load.i32 little v0
+  return v1
+}
+
+; check:  lrv %r2, 0(%r2)
+; nextln: br %r14
+
+function %atomic_load_i32_sym() -> i32 {
+  gv0 = symbol colocated %sym
+block0:
+  v0 = symbol_value.i64 gv0
+  v1 = atomic_load.i32 little v0
+  return v1
+}
+
+; check:  larl %r1, %sym + 0 ; lrv %r2, 0(%r1)
+; nextln: br %r14
+
+function %atomic_load_i16(i64) -> i16 {
+block0(v0: i64):
+  v1 = atomic_load.i16 little v0
+  return v1
+}
+
+; check:  lrvh %r2, 0(%r2)
+; nextln: br %r14
+
+function %atomic_load_i16_sym() -> i16 {
+  gv0 = symbol colocated %sym
+block0:
+  v0 = symbol_value.i64 gv0
+  v1 = atomic_load.i16 little v0
+  return v1
+}
+
+; check:  larl %r1, %sym + 0 ; lrvh %r2, 0(%r1)
+; nextln: br %r14
+
+function %atomic_load_i8(i64) -> i8 {
+block0(v0: i64):
+  v1 = atomic_load.i8 little v0
+  return v1
+}
+
+; check:  llc %r2, 0(%r2)
+; nextln: br %r14
+

cranelift/filetests/filetests/isa/s390x/atomic_load.clif

@@ -0,0 +1,72 @@
+test compile
+target s390x
+
+function %atomic_load_i64(i64) -> i64 {
+block0(v0: i64):
+  v1 = atomic_load.i64 v0
+  return v1
+}
+
+; check:  lg %r2, 0(%r2)
+; nextln: br %r14
+
+function %atomic_load_i64_sym() -> i64 {
+  gv0 = symbol colocated %sym
+block0:
+  v0 = symbol_value.i64 gv0
+  v1 = atomic_load.i64 v0
+  return v1
+}
+
+; check:  lgrl %r2, %sym + 0
+; nextln: br %r14
+
+function %atomic_load_i32(i64) -> i32 {
+block0(v0: i64):
+  v1 = atomic_load.i32 v0
+  return v1
+}
+
+; check:  l %r2, 0(%r2)
+; nextln: br %r14
+
+function %atomic_load_i32_sym() -> i32 {
+  gv0 = symbol colocated %sym
+block0:
+  v0 = symbol_value.i64 gv0
+  v1 = atomic_load.i32 v0
+  return v1
+}
+
+; check:  lrl %r2, %sym + 0
+; nextln: br %r14
+
+function %atomic_load_i16(i64) -> i16 {
+block0(v0: i64):
+  v1 = atomic_load.i16 v0
+  return v1
+}
+
+; check:  llh %r2, 0(%r2)
+; nextln: br %r14
+
+function %atomic_load_i16_sym() -> i16 {
+  gv0 = symbol colocated %sym
+block0:
+  v0 = symbol_value.i64 gv0
+  v1 = atomic_load.i16 v0
+  return v1
+}
+
+; check:  llhrl %r2, %sym + 0
+; nextln: br %r14
+
+function %atomic_load_i8(i64) -> i8 {
+block0(v0: i64):
+  v1 = atomic_load.i8 v0
+  return v1
+}
+
+; check:  llc %r2, 0(%r2)
+; nextln: br %r14
+

cranelift/filetests/filetests/isa/s390x/atomic_rmw.clif

@@ -0,0 +1,114 @@
+test compile
+target s390x
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; ATOMIC_RMW (ADD)
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+function %atomic_rmw_add_i64(i64, i64) -> i64 {
+block0(v0: i64, v1: i64):
+  v2 = atomic_rmw.i64 add v0, v1
+  return v2
+}
+
+; check:  laag %r2, %r3, 0(%r2)
+; nextln: br %r14
+
+function %atomic_rmw_add_i32(i64, i32) -> i32 {
+block0(v0: i64, v1: i32):
+  v2 = atomic_rmw.i32 add v0, v1
+  return v2
+}
+
+; check:  laa %r2, %r3, 0(%r2)
+; nextln: br %r14
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; ATOMIC_RMW (SUB)
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+function %atomic_rmw_sub_i64(i64, i64) -> i64 {
+block0(v0: i64, v1: i64):
+  v2 = atomic_rmw.i64 sub v0, v1
+  return v2
+}
+
+; check:  lcgr %r3, %r3
+; nextln: laag %r2, %r3, 0(%r2)
+; nextln: br %r14
+
+function %atomic_rmw_sub_i32(i64, i32) -> i32 {
+block0(v0: i64, v1: i32):
+  v2 = atomic_rmw.i32 sub v0, v1
+  return v2
+}
+
+; check:  lcr %r3, %r3
+; nextln: laa %r2, %r3, 0(%r2)
+; nextln: br %r14
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; ATOMIC_RMW (AND)
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+function %atomic_rmw_and_i64(i64, i64) -> i64 {
+block0(v0: i64, v1: i64):
+  v2 = atomic_rmw.i64 and v0, v1
+  return v2
+}
+
+; check:  lang %r2, %r3, 0(%r2)
+; nextln: br %r14
+
+function %atomic_rmw_and_i32(i64, i32) -> i32 {
+block0(v0: i64, v1: i32):
+  v2 = atomic_rmw.i32 and v0, v1
+  return v2
+}
+
+; check:  lan %r2, %r3, 0(%r2)
+; nextln: br %r14
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; ATOMIC_RMW (OR)
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+function %atomic_rmw_or_i64(i64, i64) -> i64 {
+block0(v0: i64, v1: i64):
+  v2 = atomic_rmw.i64 or v0, v1
+  return v2
+}
+
+; check:  laog %r2, %r3, 0(%r2)
+; nextln: br %r14
+
+function %atomic_rmw_or_i32(i64, i32) -> i32 {
+block0(v0: i64, v1: i32):
+  v2 = atomic_rmw.i32 or v0, v1
+  return v2
+}
+
+; check:  lao %r2, %r3, 0(%r2)
+; nextln: br %r14
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; ATOMIC_RMW (XOR)
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+function %atomic_rmw_xor_i64(i64, i64) -> i64 {
+block0(v0: i64, v1: i64):
+  v2 = atomic_rmw.i64 xor v0, v1
+  return v2
+}
+
+; check:  laxg %r2, %r3, 0(%r2)
+; nextln: br %r14
+
+function %atomic_rmw_xor_i32(i64, i32) -> i32 {
+block0(v0: i64, v1: i32):
+  v2 = atomic_rmw.i32 xor v0, v1
+  return v2
+}
+
+; check:  lax %r2, %r3, 0(%r2)
+; nextln: br %r14

cranelift/filetests/filetests/isa/s390x/atomic_store-little.clif

@@ -0,0 +1,125 @@
+test compile
+target s390x
+
+function %atomic_store_i64(i64, i64) {
+block0(v0: i64, v1: i64):
+  atomic_store.i64 little v0, v1
+  return
+}
+
+; check:  strvg %r2, 0(%r3)
+; nextln: bcr 14, 0
+; nextln: br %r14
+
+function %atomic_store_i64_sym(i64) {
+  gv0 = symbol colocated %sym
+block0(v0: i64):
+  v1 = symbol_value.i64 gv0
+  atomic_store.i64 little v0, v1
+  return
+}
+
+; check:  larl %r1, %sym + 0 ; strvg %r2, 0(%r1)
+; nextln: bcr 14, 0
+; nextln: br %r14
+
+function %atomic_store_imm_i64(i64) {
+block0(v0: i64):
+  v1 = iconst.i64 12345
+  atomic_store.i64 little v1, v0
+  return
+}
+
+; check:  lghi %r3, 12345
+; nextln: strvg %r3, 0(%r2)
+; nextln: bcr 14, 0
+; nextln: br %r14
+
+function %atomic_store_i32(i32, i64) {
+block0(v0: i32, v1: i64):
+  atomic_store.i32 little v0, v1
+  return
+}
+
+; check:  strv %r2, 0(%r3)
+; nextln: bcr 14, 0
+; nextln: br %r14
+
+function %atomic_store_i32_sym(i32) {
+  gv0 = symbol colocated %sym
+block0(v0: i32):
+  v1 = symbol_value.i64 gv0
+  atomic_store.i32 little v0, v1
+  return
+}
+
+; check:  larl %r1, %sym + 0 ; strv %r2, 0(%r1)
+; nextln: bcr 14, 0
+; nextln: br %r14
+
+function %atomic_store_imm_i32(i64) {
+block0(v0: i64):
+  v1 = iconst.i32 12345
+  atomic_store.i32 little v1, v0
+  return
+}
+
+; check:  lhi %r3, 12345
+; nextln: strv %r3, 0(%r2)
+; nextln: bcr 14, 0
+; nextln: br %r14
+
+function %atomic_store_i16(i16, i64) {
+block0(v0: i16, v1: i64):
+  atomic_store.i16 little v0, v1
+  return
+}
+
+; check:  strvh %r2, 0(%r3)
+; nextln: bcr 14, 0
+; nextln: br %r14
+
+function %atomic_store_i16_sym(i16) {
+  gv0 = symbol colocated %sym
+block0(v0: i16):
+  v1 = symbol_value.i64 gv0
+  atomic_store.i16 little v0, v1
+  return
+}
+
+; check:  larl %r1, %sym + 0 ; strvh %r2, 0(%r1)
+; nextln: bcr 14, 0
+; nextln: br %r14
+
+function %atomic_store_imm_i16(i64) {
+block0(v0: i64):
+  v1 = iconst.i16 12345
+  atomic_store.i16 little v1, v0
+  return
+}
+
+; check:  mvhhi 0(%r2), 14640
+; nextln: bcr 14, 0
+; nextln: br %r14
+
+function %atomic_store_i8(i8, i64) {
+block0(v0: i8, v1: i64):
+  atomic_store.i8 little v0, v1
+  return
+}
+
+; check:  stc %r2, 0(%r3)
+; nextln: bcr 14, 0
+; nextln: br %r14
+
+function %atomic_store_imm_i8(i64) {
+block0(v0: i64):
+  v1 = iconst.i8 123
+  atomic_store.i8 little v1, v0
+  return
+}
+
+; check:  mvi 0(%r2), 123
+; nextln: bcr 14, 0
+; nextln: br %r14
+

cranelift/filetests/filetests/isa/s390x/atomic_store.clif

@@ -0,0 +1,123 @@
+test compile
+target s390x
+
+function %atomic_store_i64(i64, i64) {
+block0(v0: i64, v1: i64):
+  atomic_store.i64 v0, v1
+  return
+}
+
+; check:  stg %r2, 0(%r3)
+; nextln: bcr 14, 0
+; nextln: br %r14
+
+function %atomic_store_i64_sym(i64) {
+  gv0 = symbol colocated %sym
+block0(v0: i64):
+  v1 = symbol_value.i64 gv0
+  atomic_store.i64 v0, v1
+  return
+}
+
+; check:  stgrl %r2, %sym + 0
+; nextln: bcr 14, 0
+; nextln: br %r14
+
+function %atomic_store_imm_i64(i64) {
+block0(v0: i64):
+  v1 = iconst.i64 12345
+  atomic_store.i64 v1, v0
+  return
+}
+
+; check:  mvghi 0(%r2), 12345
+; nextln: bcr 14, 0
+; nextln: br %r14
+
+function %atomic_store_i32(i32, i64) {
+block0(v0: i32, v1: i64):
+  atomic_store.i32 v0, v1
+  return
+}
+
+; check:  st %r2, 0(%r3)
+; nextln: bcr 14, 0
+; nextln: br %r14
+
+function %atomic_store_i32_sym(i32) {
+  gv0 = symbol colocated %sym
+block0(v0: i32):
+  v1 = symbol_value.i64 gv0
+  atomic_store.i32 v0, v1
+  return
+}
+
+; check:  strl %r2, %sym + 0
+; nextln: bcr 14, 0
+; nextln: br %r14
+
+function %atomic_store_imm_i32(i64) {
+block0(v0: i64):
+  v1 = iconst.i32 12345
+  atomic_store.i32 v1, v0
+  return
+}
+
+; check:  mvhi 0(%r2), 12345
+; nextln: bcr 14, 0
+; nextln: br %r14
+
+function %atomic_store_i16(i16, i64) {
+block0(v0: i16, v1: i64):
+  atomic_store.i16 v0, v1
+  return
+}
+
+; check:  sth %r2, 0(%r3)
+; nextln: bcr 14, 0
+; nextln: br %r14
+
+function %atomic_store_i16_sym(i16) {
+  gv0 = symbol colocated %sym
+block0(v0: i16):
+  v1 = symbol_value.i64 gv0
+  atomic_store.i16 v0, v1
+  return
+}
+
+; check:  sthrl %r2, %sym + 0
+; nextln: bcr 14, 0
+; nextln: br %r14
+
+function %atomic_store_imm_i16(i64) {
+block0(v0: i64):
+  v1 = iconst.i16 12345
+  atomic_store.i16 v1, v0
+  return
+}
+
+; check:  mvhhi 0(%r2), 12345
+; nextln: bcr 14, 0
+; nextln: br %r14
+
+function %atomic_store_i8(i8, i64) {
+block0(v0: i8, v1: i64):
+  atomic_store.i8 v0, v1
+  return
+}
+
+; check:  stc %r2, 0(%r3)
+; nextln: bcr 14, 0
+; nextln: br %r14
+
+function %atomic_store_imm_i8(i64) {
+block0(v0: i64):
+  v1 = iconst.i8 123
+  atomic_store.i8 v1, v0
+  return
+}
+
+; check:  mvi 0(%r2), 123
+; nextln: bcr 14, 0
+; nextln: br %r14
+

cranelift/filetests/filetests/isa/s390x/fence.clif

@@ -0,0 +1,17 @@
+test compile
+target s390x
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; FENCE
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+function %fence() {
+block0:
+  fence
+  return
+}
+
+; check:  bcr 14, 0
+; nextln: br %r14
+
+