I definitely agree that, by far, the best way to do this is to just use type punning with pointers. But if you really want to rebuild this yourself, you can still do so without the unsafe pointer, by handling signed and unsigned separately.

Unsigned is easy:

  func integerWithBytes<T: UnsignedIntegerType>(bytes: [UInt8]) -> T? {
      if bytes.count < sizeof(T.self) {
          return nil
      }
      var acc: UIntMax = 0
      for var i = 0; i < sizeof(T.self); i++ {
          acc |= bytes[i].toUIntMax() << UIntMax(i * 8)
      }
      // UnsignedIntegerType defines init(_: UIntMax)
      return T(acc)
  }

Signed is trickier because of the sign bit. You have to sign-extend manually:

  func integerWithBytes<T: SignedIntegerType>(bytes: [UInt8]) -> T? {
      if bytes.count < sizeof(T.self) {
          return nil
      }
      var acc: UIntMax = 0
      for var i = 0; i < sizeof(T.self); i++ {
          acc |= bytes[i].toUIntMax() << UIntMax(i * 8)
      }
      if sizeof(T.self) < sizeof(UIntMax.self) {
          // sign-extend the accumulator first
          if bytes[sizeof(T.self)-1] & 0x80 != 0 {
              for var i = sizeof(T.self); i < sizeof(UIntMax.self); i++ {
                  acc |= 0xFF << UIntMax(i * 8)
              }
          }
      }
      // We're assuming that IntMax is the same size as UIntMax and therefore
      // uses init(bitPattern:) to convert, but that should be safe.
      let signed = IntMax(bitPattern: acc)
      // SignedIntegerType defines init(_: IntMax)
      return T(signed)
  }