Github user vanzin commented on a diff in the pull request:
https://github.com/apache/spark/pull/5574#discussion_r29067406
--- Diff: core/src/test/scala/org/apache/spark/util/UtilsSuite.scala ---
@@ -82,6 +86,107 @@ class UtilsSuite extends FunSuite with
ResetSystemProperties {
}
}
+ test("Test byteString conversion") {
+ // Test zero
+ assert(Utils.byteStringAsBytes("0") === 0)
+
+ assert(Utils.byteStringAsGb("1") === 1)
+ assert(Utils.byteStringAsGb("1g") === 1)
+ assert(Utils.byteStringAsGb("1023m") === 0)
+ assert(Utils.byteStringAsGb("1024m") === 1)
+ assert(Utils.byteStringAsGb("1048575k") === 0)
+ assert(Utils.byteStringAsGb("1048576k") === 1)
+ assert(Utils.byteStringAsGb("1k") === 0)
+ assert(Utils.byteStringAsGb("1t") === ByteUnit.TiB.toGiB(1))
+ assert(Utils.byteStringAsGb("1p") === ByteUnit.PiB.toGiB(1))
+
+ assert(Utils.byteStringAsMb("1") === 1)
+ assert(Utils.byteStringAsMb("1m") === 1)
+ assert(Utils.byteStringAsMb("1048575b") === 0)
+ assert(Utils.byteStringAsMb("1048576b") === 1)
+ assert(Utils.byteStringAsMb("1023k") === 0)
+ assert(Utils.byteStringAsMb("1024k") === 1)
+ assert(Utils.byteStringAsMb("3645k") === 3)
+ assert(Utils.byteStringAsMb("1024gb") === 1048576)
+ assert(Utils.byteStringAsMb("1g") === ByteUnit.GiB.toMiB(1))
+ assert(Utils.byteStringAsMb("1t") === ByteUnit.TiB.toMiB(1))
+ assert(Utils.byteStringAsMb("1p") === ByteUnit.PiB.toMiB(1))
+
+ assert(Utils.byteStringAsKb("1") === 1)
+ assert(Utils.byteStringAsKb("1k") === 1)
+ assert(Utils.byteStringAsKb("1m") === ByteUnit.MiB.toKiB(1))
+ assert(Utils.byteStringAsKb("1g") === ByteUnit.GiB.toKiB(1))
+ assert(Utils.byteStringAsKb("1t") === ByteUnit.TiB.toKiB(1))
+ assert(Utils.byteStringAsKb("1p") === ByteUnit.PiB.toKiB(1))
+
+ assert(Utils.byteStringAsBytes("1") === 1)
+ assert(Utils.byteStringAsBytes("1k") === ByteUnit.KiB.toBytes(1))
+ assert(Utils.byteStringAsBytes("1m") === ByteUnit.MiB.toBytes(1))
+ assert(Utils.byteStringAsBytes("1g") === ByteUnit.GiB.toBytes(1))
+ assert(Utils.byteStringAsBytes("1t") === ByteUnit.TiB.toBytes(1))
+ assert(Utils.byteStringAsBytes("1p") === ByteUnit.PiB.toBytes(1))
+
+ // Overflow handling, 1073741824p exceeds Long.MAX_VALUE if converted
straight to Bytes
+ // This demonstrates that we can have e.g 1024^3 PB without
overflowing.
+ assert(Utils.byteStringAsGb("1073741824p") ===
ByteUnit.PiB.toGiB(1073741824))
+ assert(Utils.byteStringAsMb("1073741824p") ===
ByteUnit.PiB.toMiB(1073741824))
+
+ // Run this to confirm it doesn't throw an exception
+ assert(Utils.byteStringAsBytes("9223372036854775807") ===
9223372036854775807L)
+ assert(ByteUnit.PiB.toPiB(9223372036854775807L) ===
9223372036854775807L)
+
+ // Test overflow exception
+ intercept[IllegalArgumentException] {
+ // This value exceeds Long.MAX when converted to bytes
+ Utils.byteStringAsBytes("9223372036854775808")
+ }
+
+ // Test overflow exception
+ intercept[IllegalArgumentException] {
+ // This value exceeds Long.MAX when converted to TB
+ ByteUnit.PiB.toTiB(9223372036854775807L)
+ }
+
+ // Test fractional string
+ intercept[NumberFormatException] {
+ Utils.byteStringAsMb("0.064")
+ }
+
+ // Test fractional string
+ intercept[NumberFormatException] {
+ Utils.byteStringAsMb("0.064m")
+ }
+
+ // Test invalid strings
+ intercept[NumberFormatException] {
+ Utils.byteStringAsBytes("500ub")
+ }
+
+ // Test invalid strings
+ intercept[NumberFormatException] {
+ Utils.byteStringAsBytes("This breaks 600b")
+ }
+
+ intercept[NumberFormatException] {
+ Utils.byteStringAsBytes("This breaks 600")
+ }
+
+ intercept[NumberFormatException] {
+ Utils.byteStringAsBytes("600gb This breaks")
+ }
+
+ intercept[NumberFormatException] {
+ Utils.byteStringAsBytes("This 123mb breaks")
+ }
+
+// // Test overflow
--- End diff --
Delete these lines?
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at [email protected] or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
