Two years ago, Spotify introduced Scio, an open-source Scala framework to develop data pipelines and deploy them on Google Dataflow. In this talk, we will discuss the evolution of Scio, and share the highlights of running Scio in production for two years. We will showcase several interesting data processing workflows ran at Spotify, what we learned from running them in production, and how we leveraged that knowledge to make Scio faster, and safer and easier to use.
28. 03. 円周率
"Now I need a drink, alcoholic of course, after the
heavy lectures involving quantum mechanics."
という文を単語に分解し,
各単語の(アルファベットの)文字数を先頭か
ら出現順に並べたリストを作成せよ.
→ List(3, 1, 4, 1, 5, 9, 2, 6, 5, 3, 5, 8, 9, 7, 9)
29. 03. 円周率(Java)
@Test
public void q03() throws Exception {
String str = "Now I need a drink, alcoholic of course, after
the heavy lectures involving quantum mechanics.";
List<Integer> result = Stream.of(str.split("s+"))
.map(s -> s.replaceAll("W", ""))
.map(String::length)
.collect(Collectors.toList());
assertThat(result, contains(3, 1, 4, 1, 5, 9, 2, 6, 5, 3, 5, 8, 9,
7, 9));
}
30. 03. 円周率(Kotlin)
@Test
@Throws(Exception::class)
fun q03() {
val str = "Now I need a drink, alcoholic of course, after the
heavy lectures involving quantum mechanics."
val result = str.split("""s+""".toRegex())
.map { it.replace("""W+""".toRegex(), "") }
.map(String::length)
assertThat(result, contains(3, 1, 4, 1, 5, 9, 2, 6, 5, 3, 5, 8, 9,
7, 9))
}
31. 03. 円周率(Kotlin)
@Test
@Throws(Exception::class)
fun q03() {
val str = "Now I need a drink, alcoholic of course, after the
heavy lectures involving quantum mechanics."
val result = str.split("""s+""".toRegex())
.map { it.replace("""W+""".toRegex(), "") }
.map(String::length)
assertThat(result, contains(3, 1, 4, 1, 5, 9, 2, 6, 5, 3, 5, 8, 9,
7, 9))
}
32. 03. 円周率(Kotlin)
@Test
@Throws(Exception::class)
fun q03() {
val str = "Now I need a drink, alcoholic of course, after the
heavy lectures involving quantum mechanics."
val result = str.split("""s+""".toRegex())
.map { it.replace("""W+""".toRegex(), "") }
.map(String::length)
assertThat(result, contains(3, 1, 4, 1, 5, 9, 2, 6, 5, 3, 5, 8, 9,
7, 9))
}
33. 03. 円周率(Kotlin)
@Test
@Throws(Exception::class)
fun q03() {
val str = "Now I need a drink, alcoholic of course, after the
heavy lectures involving quantum mechanics."
val result = str.split("""s+""".toRegex())
.map { s -> s.replace("""W+""".toRegex(), "") }
.map(String::length)
assertThat(result, contains(3, 1, 4, 1, 5, 9, 2, 6, 5, 3, 5, 8, 9,
7, 9))
}
34. 04. 元素記号
"Hi He Lied Because Boron Could Not Oxidize Fluorine. New
Nations Might Also Sign Peace Security Clause. Arthur King
Can."
という文を単語に分解し,
1, 5, 6, 7, 8, 9, 15, 16, 19番目の単語は先頭の1文字,
それ以外の単語は先頭に2文字を取り出し,
取り出した文字列から単語の位置(先頭から何番目の単語か
)への連想配列(辞書型もしくはマップ型)を作成せよ.
→ Map(1 -> "H", 2 -> "He", 3 -> "Li", .... 20 -> "Ca")
35. 04. 元素記号(Java)
@Test
public void q04() throws Exception {
String str = "Hi He Lied Because Boron Could Not Oxidize Fluorine. New Nations Might Also Sign Peace
Security Clause. Arthur King Can.";
List<Integer> takeOne = Arrays.asList(0, 4, 5, 6, 7, 8, 14, 15, 18);
String[] words = str.split("s+");
Map<Integer, String> result = new LinkedHashMap<>();
for (int i = 0; i < words.length; i++) {
int take = takeOne.contains(i) ? 1 : 2;
result.put(i + 1, words[i].substring(0, take));
}
Map<Integer, String> expect = new LinkedHashMap<>();
expect.put( 1, "H");
expect.put( 2, "He");
expect.put( 3, "Li");
// ...
expect.put(20, "Ca");
assertThat(result, is(expect));
}
36. 04. 元素記号(Java)
@Test
public void q04() throws Exception {
String str = "Hi He Lied Because Boron Could Not Oxidize Fluorine.
New Nations Might Also Sign Peace Security Clause. Arthur King Can.";
List<Integer> takeOne = Arrays.asList(0, 4, 5, 6, 7, 8, 14, 15, 18);
String[] words = str.split("s+");
Map<Integer, String> result = new LinkedHashMap<>();
for (int i = 0; i < words.length; i++) {
int take = takeOne.contains(i) ? 1 : 2;
result.put(i + 1, words[i].substring(0, take));
}
// ...
}
37. @Test
@Throws(Exception::class)
fun q04() {
val text = "Hi He Lied Because Boron Could Not Oxidize Fluorine. New Nations Might Also Sign Peace
Security Clause. Arthur King Can."
val extractOne = listOf(0, 4, 5, 6, 7, 8, 14, 15, 18)
val result = text.split("""s+""".toRegex())
.mapIndexed { i, s ->
if (extractOne.contains(i)) i + 1 to s.take(1)
else i + 1 to s.take(2)
}.toMap()
val expect: Map<Int, String> = linkedMapOf(
1 to "H" ,
2 to "He",
3 to "Li",
// ...
20 to "Ca"
)
assertThat(result, `is`(expect))
}
04. 元素記号(Kotlin)
38. @Test
@Throws(Exception::class)
fun q04() {
val text = "Hi He Lied Because Boron Could Not Oxidize Fluorine. New
Nations Might Also Sign Peace Security Clause. Arthur King Can."
val extractOne = listOf(0, 4, 5, 6, 7, 8, 14, 15, 18)
val result = text.split("""s+""".toRegex())
.mapIndexed { i, s ->
if (extractOne.contains(i)) i + 1 to s.take(1)
else i + 1 to s.take(2)
}.toMap()
// ...
}
04. 元素記号(Kotlin)
39. @Test
@Throws(Exception::class)
fun q04() {
val text = "Hi He Lied Because Boron Could Not Oxidize Fluorine. New
Nations Might Also Sign Peace Security Clause. Arthur King Can."
val extractOne = listOf(0, 4, 5, 6, 7, 8, 14, 15, 18)
val result = text.split("""s+""".toRegex())
.mapIndexed { i, s ->
if (extractOne.contains(i)) i + 1 to s.take(1)
else i + 1 to s.take(2)
}.toMap()
// ...
}
04. 元素記号(Kotlin)
40. @Test
@Throws(Exception::class)
fun q04() {
val text = "Hi He Lied Because Boron Could Not Oxidize Fluorine. New
Nations Might Also Sign Peace Security Clause. Arthur King Can."
val extractOne = listOf(0, 4, 5, 6, 7, 8, 14, 15, 18)
val result = text.split("""s+""".toRegex())
.mapIndexed { i, s ->
if (extractOne.contains(i)) i + 1 to s.take(1)
else i + 1 to s.take(2)
}.toMap()
// ...
}
04. 元素記号(Kotlin)
// Tuples.kt
public infix fun <A, B> A.to(that: B): Pair<A, B> = Pair(this, that)
41. @Test
@Throws(Exception::class)
fun q04() {
val text = "Hi He Lied Because Boron Could Not Oxidize Fluorine. New
Nations Might Also Sign Peace Security Clause. Arthur King Can."
val extractOne = listOf(0, 4, 5, 6, 7, 8, 14, 15, 18)
val result = text.split("""s+""".toRegex())
.mapIndexed { i, s ->
if (extractOne.contains(i)) i + 1 to s.take(1)
else i + 1 to s.take(2)
}.toMap()
// ...
}
04. 元素記号(Kotlin)
// Maps.kt
public fun <K, V> Iterable<Pair<K, V>>.toMap(): Map<K, V>