diff --git a/python/datafusion/functions.py b/python/datafusion/functions.py index a4933a747..3fedf9ce2 100644 --- a/python/datafusion/functions.py +++ b/python/datafusion/functions.py @@ -386,8 +386,6 @@ def list_to_string(expr: Expr, delimiter: Expr) -> Expr: def list_join(expr: Expr, delimiter: Expr) -> Expr: """Converts each element to its text representation. - This is an alias for :py:func:`array_to_string`. - See Also: This is an alias for :py:func:`array_to_string`. """ @@ -407,6 +405,15 @@ def in_list(arg: Expr, values: list[Expr], negated: bool = False) -> Expr: ... ) >>> result.collect_column("in").to_pylist() [True, False, True] + + >>> result = df.select( + ... dfn.functions.in_list( + ... dfn.col("a"), [dfn.lit(1), dfn.lit(3)], + ... negated=True, + ... ).alias("not_in") + ... ) + >>> result.collect_column("not_in").to_pylist() + [False, True, False] """ values = [v.expr for v in values] return Expr(f.in_list(arg.expr, values, negated)) @@ -471,6 +478,11 @@ def order_by(expr: Expr, ascending: bool = True, nulls_first: bool = True) -> So >>> sort_expr = dfn.functions.order_by(dfn.col("a"), ascending=False) >>> sort_expr.ascending() False + + >>> sort_expr = dfn.functions.order_by( + ... dfn.col("a"), ascending=True, nulls_first=False) + >>> sort_expr.nulls_first() + False """ return SortExpr(expr, ascending=ascending, nulls_first=nulls_first) @@ -490,6 +502,13 @@ def alias(expr: Expr, name: str, metadata: dict[str, str] | None = None) -> Expr ... dfn.functions.alias(dfn.col("a"), "b") ... ).collect_column("b")[0].as_py() 1 + + >>> df.select( + ... dfn.functions.alias( + ... dfn.col("a"), "b", metadata={"info": "test"} + ... ) + ... ).collect_column("b")[0].as_py() + 1 """ return Expr(f.alias(expr.expr, name, metadata)) @@ -523,6 +542,13 @@ def count_star(filter: Expr | None = None) -> Expr: >>> result = df.aggregate([], [dfn.functions.count_star().alias("cnt")]) >>> result.collect_column("cnt")[0].as_py() 3 + + >>> result = df.aggregate( + ... [], [dfn.functions.count_star( + ... filter=dfn.col("a") > dfn.lit(1) + ... ).alias("cnt")]) + >>> result.collect_column("cnt")[0].as_py() + 2 """ return count(Expr.literal(1), filter=filter) @@ -652,12 +678,11 @@ def ascii(arg: Expr) -> Expr: """Returns the numeric code of the first character of the argument. Examples: - --------- - >>> ctx = dfn.SessionContext() - >>> df = ctx.from_pydict({"a": ["a","b","c"]}) - >>> ascii_df = df.select(dfn.functions.ascii(dfn.col("a")).alias("ascii")) - >>> ascii_df.collect_column("ascii")[0].as_py() - 97 + >>> ctx = dfn.SessionContext() + >>> df = ctx.from_pydict({"a": ["a","b","c"]}) + >>> ascii_df = df.select(dfn.functions.ascii(dfn.col("a")).alias("ascii")) + >>> ascii_df.collect_column("ascii")[0].as_py() + 97 """ return Expr(f.ascii(arg.expr)) @@ -732,12 +757,11 @@ def bit_length(arg: Expr) -> Expr: """Returns the number of bits in the string argument. Examples: - --------- - >>> ctx = dfn.SessionContext() - >>> df = ctx.from_pydict({"a": ["a","b","c"]}) - >>> bit_df = df.select(dfn.functions.bit_length(dfn.col("a")).alias("bit_len")) - >>> bit_df.collect_column("bit_len")[0].as_py() - 8 + >>> ctx = dfn.SessionContext() + >>> df = ctx.from_pydict({"a": ["a","b","c"]}) + >>> bit_df = df.select(dfn.functions.bit_length(dfn.col("a")).alias("bit_len")) + >>> bit_df.collect_column("bit_len")[0].as_py() + 8 """ return Expr(f.bit_length(arg.expr)) @@ -746,12 +770,11 @@ def btrim(arg: Expr) -> Expr: """Removes all characters, spaces by default, from both sides of a string. Examples: - --------- - >>> ctx = dfn.SessionContext() - >>> df = ctx.from_pydict({"a": [" a "]}) - >>> trim_df = df.select(dfn.functions.btrim(dfn.col("a")).alias("trimmed")) - >>> trim_df.collect_column("trimmed")[0].as_py() - 'a' + >>> ctx = dfn.SessionContext() + >>> df = ctx.from_pydict({"a": [" a "]}) + >>> trim_df = df.select(dfn.functions.btrim(dfn.col("a")).alias("trimmed")) + >>> trim_df.collect_column("trimmed")[0].as_py() + 'a' """ return Expr(f.btrim(arg.expr)) @@ -786,13 +809,12 @@ def character_length(arg: Expr) -> Expr: """Returns the number of characters in the argument. Examples: - --------- - >>> ctx = dfn.SessionContext() - >>> df = ctx.from_pydict({"a": ["abc","b","c"]}) - >>> char_len_df = df.select( - ... dfn.functions.character_length(dfn.col("a")).alias("char_len")) - >>> char_len_df.collect_column("char_len")[0].as_py() - 3 + >>> ctx = dfn.SessionContext() + >>> df = ctx.from_pydict({"a": ["abc","b","c"]}) + >>> char_len_df = df.select( + ... dfn.functions.character_length(dfn.col("a")).alias("char_len")) + >>> char_len_df.collect_column("char_len")[0].as_py() + 3 """ return Expr(f.character_length(arg.expr)) @@ -801,12 +823,11 @@ def length(string: Expr) -> Expr: """The number of characters in the ``string``. Examples: - --------- - >>> ctx = dfn.SessionContext() - >>> df = ctx.from_pydict({"a": ["hello"]}) - >>> result = df.select(dfn.functions.length(dfn.col("a")).alias("len")) - >>> result.collect_column("len")[0].as_py() - 5 + >>> ctx = dfn.SessionContext() + >>> df = ctx.from_pydict({"a": ["hello"]}) + >>> result = df.select(dfn.functions.length(dfn.col("a")).alias("len")) + >>> result.collect_column("len")[0].as_py() + 5 """ return Expr(f.length(string.expr)) @@ -815,12 +836,11 @@ def char_length(string: Expr) -> Expr: """The number of characters in the ``string``. Examples: - --------- - >>> ctx = dfn.SessionContext() - >>> df = ctx.from_pydict({"a": ["hello"]}) - >>> result = df.select(dfn.functions.char_length(dfn.col("a")).alias("len")) - >>> result.collect_column("len")[0].as_py() - 5 + >>> ctx = dfn.SessionContext() + >>> df = ctx.from_pydict({"a": ["hello"]}) + >>> result = df.select(dfn.functions.char_length(dfn.col("a")).alias("len")) + >>> result.collect_column("len")[0].as_py() + 5 """ return Expr(f.char_length(string.expr)) @@ -829,12 +849,11 @@ def chr(arg: Expr) -> Expr: """Converts the Unicode code point to a UTF8 character. Examples: - --------- - >>> ctx = dfn.SessionContext() - >>> df = ctx.from_pydict({"a": [65]}) - >>> result = df.select(dfn.functions.chr(dfn.col("a")).alias("chr")) - >>> result.collect_column("chr")[0].as_py() - 'A' + >>> ctx = dfn.SessionContext() + >>> df = ctx.from_pydict({"a": [65]}) + >>> result = df.select(dfn.functions.chr(dfn.col("a")).alias("chr")) + >>> result.collect_column("chr")[0].as_py() + 'A' """ return Expr(f.chr(arg.expr)) @@ -914,13 +933,12 @@ def ends_with(arg: Expr, suffix: Expr) -> Expr: """Returns true if the ``string`` ends with the ``suffix``, false otherwise. Examples: - --------- - >>> ctx = dfn.SessionContext() - >>> df = ctx.from_pydict({"a": ["abc","b","c"]}) - >>> ends_with_df = df.select( - ... dfn.functions.ends_with(dfn.col("a"), dfn.lit("c")).alias("ends_with")) - >>> ends_with_df.collect_column("ends_with")[0].as_py() - True + >>> ctx = dfn.SessionContext() + >>> df = ctx.from_pydict({"a": ["abc","b","c"]}) + >>> ends_with_df = df.select( + ... dfn.functions.ends_with(dfn.col("a"), dfn.lit("c")).alias("ends_with")) + >>> ends_with_df.collect_column("ends_with")[0].as_py() + True """ return Expr(f.ends_with(arg.expr, suffix.expr)) @@ -962,13 +980,12 @@ def find_in_set(string: Expr, string_list: Expr) -> Expr: The string list is a string composed of substrings separated by ``,`` characters. Examples: - --------- - >>> ctx = dfn.SessionContext() - >>> df = ctx.from_pydict({"a": ["b"]}) - >>> result = df.select( - ... dfn.functions.find_in_set(dfn.col("a"), dfn.lit("a,b,c")).alias("pos")) - >>> result.collect_column("pos")[0].as_py() - 2 + >>> ctx = dfn.SessionContext() + >>> df = ctx.from_pydict({"a": ["b"]}) + >>> result = df.select( + ... dfn.functions.find_in_set(dfn.col("a"), dfn.lit("a,b,c")).alias("pos")) + >>> result.collect_column("pos")[0].as_py() + 2 """ return Expr(f.find_in_set(string.expr, string_list.expr)) @@ -1008,12 +1025,11 @@ def initcap(string: Expr) -> Expr: characters to lowercase. Examples: - --------- - >>> ctx = dfn.SessionContext() - >>> df = ctx.from_pydict({"a": ["the cat"]}) - >>> cap_df = df.select(dfn.functions.initcap(dfn.col("a")).alias("cap")) - >>> cap_df.collect_column("cap")[0].as_py() - 'The Cat' + >>> ctx = dfn.SessionContext() + >>> df = ctx.from_pydict({"a": ["the cat"]}) + >>> cap_df = df.select(dfn.functions.initcap(dfn.col("a")).alias("cap")) + >>> cap_df.collect_column("cap")[0].as_py() + 'The Cat' """ return Expr(f.initcap(string.expr)) @@ -1021,7 +1037,8 @@ def initcap(string: Expr) -> Expr: def instr(string: Expr, substring: Expr) -> Expr: """Finds the position from where the ``substring`` matches the ``string``. - This is an alias for :py:func:`strpos`. + See Also: + This is an alias for :py:func:`strpos`. """ return strpos(string, substring) @@ -1058,12 +1075,12 @@ def left(string: Expr, n: Expr) -> Expr: """Returns the first ``n`` characters in the ``string``. Examples: - --------- - >>> ctx = dfn.SessionContext() - >>> df = ctx.from_pydict({"a": ["the cat"]}) - >>> left_df = df.select(dfn.functions.left(dfn.col("a"), dfn.lit(3)).alias("left")) - >>> left_df.collect_column("left")[0].as_py() - 'the' + >>> ctx = dfn.SessionContext() + >>> df = ctx.from_pydict({"a": ["the cat"]}) + >>> left_df = df.select( + ... dfn.functions.left(dfn.col("a"), dfn.lit(3)).alias("left")) + >>> left_df.collect_column("left")[0].as_py() + 'the' """ return Expr(f.left(string.expr, n.expr)) @@ -1072,13 +1089,12 @@ def levenshtein(string1: Expr, string2: Expr) -> Expr: """Returns the Levenshtein distance between the two given strings. Examples: - --------- - >>> ctx = dfn.SessionContext() - >>> df = ctx.from_pydict({"a": ["kitten"]}) - >>> result = df.select( - ... dfn.functions.levenshtein(dfn.col("a"), dfn.lit("sitting")).alias("d")) - >>> result.collect_column("d")[0].as_py() - 3 + >>> ctx = dfn.SessionContext() + >>> df = ctx.from_pydict({"a": ["kitten"]}) + >>> result = df.select( + ... dfn.functions.levenshtein(dfn.col("a"), dfn.lit("sitting")).alias("d")) + >>> result.collect_column("d")[0].as_py() + 3 """ return Expr(f.levenshtein(string1.expr, string2.expr)) @@ -1141,12 +1157,11 @@ def lower(arg: Expr) -> Expr: """Converts a string to lowercase. Examples: - --------- - >>> ctx = dfn.SessionContext() - >>> df = ctx.from_pydict({"a": ["THE CaT"]}) - >>> lower_df = df.select(dfn.functions.lower(dfn.col("a")).alias("lower")) - >>> lower_df.collect_column("lower")[0].as_py() - 'the cat' + >>> ctx = dfn.SessionContext() + >>> df = ctx.from_pydict({"a": ["THE CaT"]}) + >>> lower_df = df.select(dfn.functions.lower(dfn.col("a")).alias("lower")) + >>> lower_df.collect_column("lower")[0].as_py() + 'the cat' """ return Expr(f.lower(arg.expr)) @@ -1159,14 +1174,21 @@ def lpad(string: Expr, count: Expr, characters: Expr | None = None) -> Expr: truncated (on the right). Examples: - --------- - >>> ctx = dfn.SessionContext() - >>> df = ctx.from_pydict({"a": ["the cat", "a hat"]}) - >>> lpad_df = df.select(dfn.functions.lpad(dfn.col("a"), dfn.lit(6)).alias("lpad")) - >>> lpad_df.collect_column("lpad")[0].as_py() - 'the ca' - >>> lpad_df.collect_column("lpad")[1].as_py() - ' a hat' + >>> ctx = dfn.SessionContext() + >>> df = ctx.from_pydict({"a": ["the cat", "a hat"]}) + >>> lpad_df = df.select( + ... dfn.functions.lpad(dfn.col("a"), dfn.lit(6)).alias("lpad")) + >>> lpad_df.collect_column("lpad")[0].as_py() + 'the ca' + >>> lpad_df.collect_column("lpad")[1].as_py() + ' a hat' + + >>> result = df.select( + ... dfn.functions.lpad( + ... dfn.col("a"), dfn.lit(10), dfn.lit(".") + ... ).alias("lpad")) + >>> result.collect_column("lpad")[0].as_py() + '...the cat' """ characters = characters if characters is not None else Expr.literal(" ") return Expr(f.lpad(string.expr, count.expr, characters.expr)) @@ -1176,12 +1198,11 @@ def ltrim(arg: Expr) -> Expr: """Removes all characters, spaces by default, from the beginning of a string. Examples: - --------- - >>> ctx = dfn.SessionContext() - >>> df = ctx.from_pydict({"a": [" a "]}) - >>> trim_df = df.select(dfn.functions.ltrim(dfn.col("a")).alias("trimmed")) - >>> trim_df.collect_column("trimmed")[0].as_py() - 'a ' + >>> ctx = dfn.SessionContext() + >>> df = ctx.from_pydict({"a": [" a "]}) + >>> trim_df = df.select(dfn.functions.ltrim(dfn.col("a")).alias("trimmed")) + >>> trim_df.collect_column("trimmed")[0].as_py() + 'a ' """ return Expr(f.ltrim(arg.expr)) @@ -1236,12 +1257,11 @@ def octet_length(arg: Expr) -> Expr: """Returns the number of bytes of a string. Examples: - --------- - >>> ctx = dfn.SessionContext() - >>> df = ctx.from_pydict({"a": ["hello"]}) - >>> result = df.select(dfn.functions.octet_length(dfn.col("a")).alias("len")) - >>> result.collect_column("len")[0].as_py() - 5 + >>> ctx = dfn.SessionContext() + >>> df = ctx.from_pydict({"a": ["hello"]}) + >>> result = df.select(dfn.functions.octet_length(dfn.col("a")).alias("len")) + >>> result.collect_column("len")[0].as_py() + 5 """ return Expr(f.octet_length(arg.expr)) @@ -1255,14 +1275,13 @@ def overlay( extends for ``length`` characters with new substring. Examples: - --------- - >>> ctx = dfn.SessionContext() - >>> df = ctx.from_pydict({"a": ["abcdef"]}) - >>> result = df.select( - ... dfn.functions.overlay(dfn.col("a"), dfn.lit("XY"), dfn.lit(3), - ... dfn.lit(2)).alias("o")) - >>> result.collect_column("o")[0].as_py() - 'abXYef' + >>> ctx = dfn.SessionContext() + >>> df = ctx.from_pydict({"a": ["abcdef"]}) + >>> result = df.select( + ... dfn.functions.overlay(dfn.col("a"), dfn.lit("XY"), dfn.lit(3), + ... dfn.lit(2)).alias("o")) + >>> result.collect_column("o")[0].as_py() + 'abXYef' """ if length is None: return Expr(f.overlay(string.expr, substring.expr, start.expr)) @@ -1275,14 +1294,12 @@ def pi() -> Expr: Examples: >>> ctx = dfn.SessionContext() >>> df = ctx.from_pydict({"a": [1]}) - >>> import builtins + >>> from math import pi >>> result = df.select( ... dfn.functions.pi().alias("pi") ... ) - >>> builtins.round( - ... result.collect_column("pi")[0].as_py(), 5 - ... ) - 3.14159 + >>> result.collect_column("pi")[0].as_py() == pi + True """ return Expr(f.pi()) @@ -1290,7 +1307,8 @@ def pi() -> Expr: def position(string: Expr, substring: Expr) -> Expr: """Finds the position from where the ``substring`` matches the ``string``. - This is an alias for :py:func:`strpos`. + See Also: + This is an alias for :py:func:`strpos`. """ return strpos(string, substring) @@ -1313,7 +1331,8 @@ def power(base: Expr, exponent: Expr) -> Expr: def pow(base: Expr, exponent: Expr) -> Expr: """Returns ``base`` raised to the power of ``exponent``. - This is an alias of :py:func:`power`. + See Also: + This is an alias of :py:func:`power`. """ return power(base, exponent) @@ -1350,6 +1369,17 @@ def regexp_like(string: Expr, regex: Expr, flags: Expr | None = None) -> Expr: ... ) >>> result.collect_column("m")[0].as_py() True + + Use ``flags`` for case-insensitive matching: + + >>> result = df.select( + ... dfn.functions.regexp_like( + ... dfn.col("a"), dfn.lit("HELLO"), + ... flags=dfn.lit("i"), + ... ).alias("m") + ... ) + >>> result.collect_column("m")[0].as_py() + True """ if flags is not None: flags = flags.expr @@ -1372,6 +1402,17 @@ def regexp_match(string: Expr, regex: Expr, flags: Expr | None = None) -> Expr: ... ) >>> result.collect_column("m")[0].as_py() ['42'] + + Use ``flags`` for case-insensitive matching: + + >>> result = df.select( + ... dfn.functions.regexp_match( + ... dfn.col("a"), dfn.lit("(HELLO)"), + ... flags=dfn.lit("i"), + ... ).alias("m") + ... ) + >>> result.collect_column("m")[0].as_py() + ['hello'] """ if flags is not None: flags = flags.expr @@ -1400,6 +1441,18 @@ def regexp_replace( ... ) >>> result.collect_column("r")[0].as_py() 'hello XX' + + Use the ``g`` flag to replace all occurrences: + + >>> df = ctx.from_pydict({"a": ["a1 b2 c3"]}) + >>> result = df.select( + ... dfn.functions.regexp_replace( + ... dfn.col("a"), dfn.lit("\\d+"), + ... dfn.lit("X"), flags=dfn.lit("g"), + ... ).alias("r") + ... ) + >>> result.collect_column("r")[0].as_py() + 'aX bX cX' """ if flags is not None: flags = flags.expr @@ -1421,6 +1474,17 @@ def regexp_count( ... dfn.functions.regexp_count(dfn.col("a"), dfn.lit("abc")).alias("c")) >>> result.collect_column("c")[0].as_py() 2 + + Use ``start`` to begin searching from a position, and + ``flags`` for case-insensitive matching: + + >>> result = df.select( + ... dfn.functions.regexp_count( + ... dfn.col("a"), dfn.lit("ABC"), + ... start=dfn.lit(4), flags=dfn.lit("i"), + ... ).alias("c")) + >>> result.collect_column("c")[0].as_py() + 1 """ if flags is not None: flags = flags.expr @@ -1456,6 +1520,31 @@ def regexp_instr( ... ) >>> result.collect_column("pos")[0].as_py() 7 + + Use ``start`` to search from a position, ``n`` for the + nth occurrence, and ``flags`` for case-insensitive mode: + + >>> df = ctx.from_pydict({"a": ["abc ABC abc"]}) + >>> result = df.select( + ... dfn.functions.regexp_instr( + ... dfn.col("a"), dfn.lit("abc"), + ... start=dfn.lit(2), n=dfn.lit(1), + ... flags=dfn.lit("i"), + ... ).alias("pos") + ... ) + >>> result.collect_column("pos")[0].as_py() + 5 + + Use ``sub_expr`` to get the position of a capture group: + + >>> result = df.select( + ... dfn.functions.regexp_instr( + ... dfn.col("a"), dfn.lit("(abc)"), + ... sub_expr=dfn.lit(1), + ... ).alias("pos") + ... ) + >>> result.collect_column("pos")[0].as_py() + 1 """ start = start.expr if start is not None else None n = n.expr if n is not None else None @@ -1478,12 +1567,12 @@ def repeat(string: Expr, n: Expr) -> Expr: """Repeats the ``string`` to ``n`` times. Examples: - --------- - >>> ctx = dfn.SessionContext() - >>> df = ctx.from_pydict({"a": ["ha"]}) - >>> result = df.select(dfn.functions.repeat(dfn.col("a"), dfn.lit(3)).alias("r")) - >>> result.collect_column("r")[0].as_py() - 'hahaha' + >>> ctx = dfn.SessionContext() + >>> df = ctx.from_pydict({"a": ["ha"]}) + >>> result = df.select( + ... dfn.functions.repeat(dfn.col("a"), dfn.lit(3)).alias("r")) + >>> result.collect_column("r")[0].as_py() + 'hahaha' """ return Expr(f.repeat(string.expr, n.expr)) @@ -1492,14 +1581,13 @@ def replace(string: Expr, from_val: Expr, to_val: Expr) -> Expr: """Replaces all occurrences of ``from_val`` with ``to_val`` in the ``string``. Examples: - --------- - >>> ctx = dfn.SessionContext() - >>> df = ctx.from_pydict({"a": ["hello world"]}) - >>> result = df.select( - ... dfn.functions.replace(dfn.col("a"), dfn.lit("world"), - ... dfn.lit("there")).alias("r")) - >>> result.collect_column("r")[0].as_py() - 'hello there' + >>> ctx = dfn.SessionContext() + >>> df = ctx.from_pydict({"a": ["hello world"]}) + >>> result = df.select( + ... dfn.functions.replace(dfn.col("a"), dfn.lit("world"), + ... dfn.lit("there")).alias("r")) + >>> result.collect_column("r")[0].as_py() + 'hello there' """ return Expr(f.replace(string.expr, from_val.expr, to_val.expr)) @@ -1508,12 +1596,11 @@ def reverse(arg: Expr) -> Expr: """Reverse the string argument. Examples: - --------- - >>> ctx = dfn.SessionContext() - >>> df = ctx.from_pydict({"a": ["hello"]}) - >>> result = df.select(dfn.functions.reverse(dfn.col("a")).alias("r")) - >>> result.collect_column("r")[0].as_py() - 'olleh' + >>> ctx = dfn.SessionContext() + >>> df = ctx.from_pydict({"a": ["hello"]}) + >>> result = df.select(dfn.functions.reverse(dfn.col("a")).alias("r")) + >>> result.collect_column("r")[0].as_py() + 'olleh' """ return Expr(f.reverse(arg.expr)) @@ -1522,12 +1609,11 @@ def right(string: Expr, n: Expr) -> Expr: """Returns the last ``n`` characters in the ``string``. Examples: - --------- - >>> ctx = dfn.SessionContext() - >>> df = ctx.from_pydict({"a": ["hello"]}) - >>> result = df.select(dfn.functions.right(dfn.col("a"), dfn.lit(3)).alias("r")) - >>> result.collect_column("r")[0].as_py() - 'llo' + >>> ctx = dfn.SessionContext() + >>> df = ctx.from_pydict({"a": ["hello"]}) + >>> result = df.select(dfn.functions.right(dfn.col("a"), dfn.lit(3)).alias("r")) + >>> result.collect_column("r")[0].as_py() + 'llo' """ return Expr(f.right(string.expr, n.expr)) @@ -1558,13 +1644,12 @@ def rpad(string: Expr, count: Expr, characters: Expr | None = None) -> Expr: by default). If the string is already longer than length then it is truncated. Examples: - --------- - >>> ctx = dfn.SessionContext() - >>> df = ctx.from_pydict({"a": ["hi"]}) - >>> result = df.select( - ... dfn.functions.rpad(dfn.col("a"), dfn.lit(5), dfn.lit("!")).alias("r")) - >>> result.collect_column("r")[0].as_py() - 'hi!!!' + >>> ctx = dfn.SessionContext() + >>> df = ctx.from_pydict({"a": ["hi"]}) + >>> result = df.select( + ... dfn.functions.rpad(dfn.col("a"), dfn.lit(5), dfn.lit("!")).alias("r")) + >>> result.collect_column("r")[0].as_py() + 'hi!!!' """ characters = characters if characters is not None else Expr.literal(" ") return Expr(f.rpad(string.expr, count.expr, characters.expr)) @@ -1574,12 +1659,11 @@ def rtrim(arg: Expr) -> Expr: """Removes all characters, spaces by default, from the end of a string. Examples: - --------- - >>> ctx = dfn.SessionContext() - >>> df = ctx.from_pydict({"a": [" a "]}) - >>> trim_df = df.select(dfn.functions.rtrim(dfn.col("a")).alias("trimmed")) - >>> trim_df.collect_column("trimmed")[0].as_py() - ' a' + >>> ctx = dfn.SessionContext() + >>> df = ctx.from_pydict({"a": [" a "]}) + >>> trim_df = df.select(dfn.functions.rtrim(dfn.col("a")).alias("trimmed")) + >>> trim_df.collect_column("trimmed")[0].as_py() + ' a' """ return Expr(f.rtrim(arg.expr)) @@ -1690,13 +1774,14 @@ def split_part(string: Expr, delimiter: Expr, index: Expr) -> Expr: on the index. Examples: - --------- - >>> ctx = dfn.SessionContext() - >>> df = ctx.from_pydict({"a": ["a,b,c"]}) - >>> result = df.select( - ... dfn.functions.split_part(dfn.col("a"), dfn.lit(","), dfn.lit(2)).alias("s")) - >>> result.collect_column("s")[0].as_py() - 'b' + >>> ctx = dfn.SessionContext() + >>> df = ctx.from_pydict({"a": ["a,b,c"]}) + >>> result = df.select( + ... dfn.functions.split_part( + ... dfn.col("a"), dfn.lit(","), dfn.lit(2) + ... ).alias("s")) + >>> result.collect_column("s")[0].as_py() + 'b' """ return Expr(f.split_part(string.expr, delimiter.expr, index.expr)) @@ -1718,13 +1803,12 @@ def starts_with(string: Expr, prefix: Expr) -> Expr: """Returns true if string starts with prefix. Examples: - --------- - >>> ctx = dfn.SessionContext() - >>> df = ctx.from_pydict({"a": ["hello_from_datafusion"]}) - >>> result = df.select( - ... dfn.functions.starts_with(dfn.col("a"), dfn.lit("hello")).alias("sw")) - >>> result.collect_column("sw")[0].as_py() - True + >>> ctx = dfn.SessionContext() + >>> df = ctx.from_pydict({"a": ["hello_from_datafusion"]}) + >>> result = df.select( + ... dfn.functions.starts_with(dfn.col("a"), dfn.lit("hello")).alias("sw")) + >>> result.collect_column("sw")[0].as_py() + True """ return Expr(f.starts_with(string.expr, prefix.expr)) @@ -1733,13 +1817,12 @@ def strpos(string: Expr, substring: Expr) -> Expr: """Finds the position from where the ``substring`` matches the ``string``. Examples: - --------- - >>> ctx = dfn.SessionContext() - >>> df = ctx.from_pydict({"a": ["hello"]}) - >>> result = df.select( - ... dfn.functions.strpos(dfn.col("a"), dfn.lit("llo")).alias("pos")) - >>> result.collect_column("pos")[0].as_py() - 3 + >>> ctx = dfn.SessionContext() + >>> df = ctx.from_pydict({"a": ["hello"]}) + >>> result = df.select( + ... dfn.functions.strpos(dfn.col("a"), dfn.lit("llo")).alias("pos")) + >>> result.collect_column("pos")[0].as_py() + 3 """ return Expr(f.strpos(string.expr, substring.expr)) @@ -1748,12 +1831,12 @@ def substr(string: Expr, position: Expr) -> Expr: """Substring from the ``position`` to the end. Examples: - --------- - >>> ctx = dfn.SessionContext() - >>> df = ctx.from_pydict({"a": ["hello"]}) - >>> result = df.select(dfn.functions.substr(dfn.col("a"), dfn.lit(3)).alias("s")) - >>> result.collect_column("s")[0].as_py() - 'llo' + >>> ctx = dfn.SessionContext() + >>> df = ctx.from_pydict({"a": ["hello"]}) + >>> result = df.select( + ... dfn.functions.substr(dfn.col("a"), dfn.lit(3)).alias("s")) + >>> result.collect_column("s")[0].as_py() + 'llo' """ return Expr(f.substr(string.expr, position.expr)) @@ -1765,14 +1848,13 @@ def substr_index(string: Expr, delimiter: Expr, count: Expr) -> Expr: ``delimiter``. Examples: - --------- - >>> ctx = dfn.SessionContext() - >>> df = ctx.from_pydict({"a": ["a.b.c"]}) - >>> result = df.select( - ... dfn.functions.substr_index(dfn.col("a"), dfn.lit("."), - ... dfn.lit(2)).alias("s")) - >>> result.collect_column("s")[0].as_py() - 'a.b' + >>> ctx = dfn.SessionContext() + >>> df = ctx.from_pydict({"a": ["a.b.c"]}) + >>> result = df.select( + ... dfn.functions.substr_index(dfn.col("a"), dfn.lit("."), + ... dfn.lit(2)).alias("s")) + >>> result.collect_column("s")[0].as_py() + 'a.b' """ return Expr(f.substr_index(string.expr, delimiter.expr, count.expr)) @@ -1781,13 +1863,14 @@ def substring(string: Expr, position: Expr, length: Expr) -> Expr: """Substring from the ``position`` with ``length`` characters. Examples: - --------- - >>> ctx = dfn.SessionContext() - >>> df = ctx.from_pydict({"a": ["hello world"]}) - >>> result = df.select( - ... dfn.functions.substring(dfn.col("a"), dfn.lit(1), dfn.lit(5)).alias("s")) - >>> result.collect_column("s")[0].as_py() - 'hello' + >>> ctx = dfn.SessionContext() + >>> df = ctx.from_pydict({"a": ["hello world"]}) + >>> result = df.select( + ... dfn.functions.substring( + ... dfn.col("a"), dfn.lit(1), dfn.lit(5) + ... ).alias("s")) + >>> result.collect_column("s")[0].as_py() + 'hello' """ return Expr(f.substring(string.expr, position.expr, length.expr)) @@ -1822,12 +1905,11 @@ def to_hex(arg: Expr) -> Expr: """Converts an integer to a hexadecimal string. Examples: - --------- - >>> ctx = dfn.SessionContext() - >>> df = ctx.from_pydict({"a": [255]}) - >>> result = df.select(dfn.functions.to_hex(dfn.col("a")).alias("hex")) - >>> result.collect_column("hex")[0].as_py() - 'ff' + >>> ctx = dfn.SessionContext() + >>> df = ctx.from_pydict({"a": [255]}) + >>> result = df.select(dfn.functions.to_hex(dfn.col("a")).alias("hex")) + >>> result.collect_column("hex")[0].as_py() + 'ff' """ return Expr(f.to_hex(arg.expr)) @@ -1859,6 +1941,18 @@ def to_char(arg: Expr, formatter: Expr) -> Expr: For usage of ``formatter`` see the rust chrono package ``strftime`` package. [Documentation here.](https://docs.rs/chrono/latest/chrono/format/strftime/index.html) + + Examples: + >>> ctx = dfn.SessionContext() + >>> df = ctx.from_pydict({"a": ["2021-01-01T00:00:00"]}) + >>> result = df.select( + ... dfn.functions.to_char( + ... dfn.functions.to_timestamp(dfn.col("a")), + ... dfn.lit("%Y/%m/%d"), + ... ).alias("formatted") + ... ) + >>> result.collect_column("formatted")[0].as_py() + '2021/01/01' """ return Expr(f.to_char(arg.expr, formatter.expr)) @@ -1878,6 +1972,14 @@ def to_date(arg: Expr, *formatters: Expr) -> Expr: For usage of ``formatters`` see the rust chrono package ``strftime`` package. [Documentation here.](https://docs.rs/chrono/latest/chrono/format/strftime/index.html) + + Examples: + >>> ctx = dfn.SessionContext() + >>> df = ctx.from_pydict({"a": ["2021-07-20"]}) + >>> result = df.select( + ... dfn.functions.to_date(dfn.col("a")).alias("dt")) + >>> str(result.collect_column("dt")[0].as_py()) + '2021-07-20' """ return Expr(f.to_date(arg.expr, *_unwrap_exprs(formatters))) @@ -1899,6 +2001,14 @@ def to_time(arg: Expr, *formatters: Expr) -> Expr: For usage of ``formatters`` see the rust chrono package ``strftime`` package. [Documentation here.](https://docs.rs/chrono/latest/chrono/format/strftime/index.html) + + Examples: + >>> ctx = dfn.SessionContext() + >>> df = ctx.from_pydict({"a": ["14:30:00"]}) + >>> result = df.select( + ... dfn.functions.to_time(dfn.col("a")).alias("t")) + >>> str(result.collect_column("t")[0].as_py()) + '14:30:00' """ return Expr(f.to_time(arg.expr, *_unwrap_exprs(formatters))) @@ -2053,7 +2163,8 @@ def current_time() -> Expr: def datepart(part: Expr, date: Expr) -> Expr: """Return a specified part of a date. - This is an alias for :py:func:`date_part`. + See Also: + This is an alias for :py:func:`date_part`. """ return date_part(part, date) @@ -2076,7 +2187,8 @@ def date_part(part: Expr, date: Expr) -> Expr: def extract(part: Expr, date: Expr) -> Expr: """Extracts a subfield from the date. - This is an alias for :py:func:`date_part`. + See Also: + This is an alias for :py:func:`date_part`. """ return date_part(part, date) @@ -2102,7 +2214,8 @@ def date_trunc(part: Expr, date: Expr) -> Expr: def datetrunc(part: Expr, date: Expr) -> Expr: """Truncates the date to a specified level of precision. - This is an alias for :py:func:`date_trunc`. + See Also: + This is an alias for :py:func:`date_trunc`. """ return date_trunc(part, date) @@ -2148,14 +2261,13 @@ def translate(string: Expr, from_val: Expr, to_val: Expr) -> Expr: """Replaces the characters in ``from_val`` with the counterpart in ``to_val``. Examples: - --------- - >>> ctx = dfn.SessionContext() - >>> df = ctx.from_pydict({"a": ["hello"]}) - >>> result = df.select( - ... dfn.functions.translate(dfn.col("a"), dfn.lit("helo"), - ... dfn.lit("HELO")).alias("t")) - >>> result.collect_column("t")[0].as_py() - 'HELLO' + >>> ctx = dfn.SessionContext() + >>> df = ctx.from_pydict({"a": ["hello"]}) + >>> result = df.select( + ... dfn.functions.translate(dfn.col("a"), dfn.lit("helo"), + ... dfn.lit("HELO")).alias("t")) + >>> result.collect_column("t")[0].as_py() + 'HELLO' """ return Expr(f.translate(string.expr, from_val.expr, to_val.expr)) @@ -2164,12 +2276,11 @@ def trim(arg: Expr) -> Expr: """Removes all characters, spaces by default, from both sides of a string. Examples: - --------- - >>> ctx = dfn.SessionContext() - >>> df = ctx.from_pydict({"a": [" hello "]}) - >>> result = df.select(dfn.functions.trim(dfn.col("a")).alias("t")) - >>> result.collect_column("t")[0].as_py() - 'hello' + >>> ctx = dfn.SessionContext() + >>> df = ctx.from_pydict({"a": [" hello "]}) + >>> result = df.select(dfn.functions.trim(dfn.col("a")).alias("t")) + >>> result.collect_column("t")[0].as_py() + 'hello' """ return Expr(f.trim(arg.expr)) @@ -2183,6 +2294,13 @@ def trunc(num: Expr, precision: Expr | None = None) -> Expr: >>> result = df.select(dfn.functions.trunc(dfn.col("a")).alias("t")) >>> result.collect_column("t")[0].as_py() 1.0 + + >>> result = df.select( + ... dfn.functions.trunc( + ... dfn.col("a"), precision=dfn.lit(2) + ... ).alias("t")) + >>> result.collect_column("t")[0].as_py() + 1.56 """ if precision is not None: return Expr(f.trunc(num.expr, precision.expr)) @@ -2193,12 +2311,11 @@ def upper(arg: Expr) -> Expr: """Converts a string to uppercase. Examples: - --------- - >>> ctx = dfn.SessionContext() - >>> df = ctx.from_pydict({"a": ["hello"]}) - >>> result = df.select(dfn.functions.upper(dfn.col("a")).alias("u")) - >>> result.collect_column("u")[0].as_py() - 'HELLO' + >>> ctx = dfn.SessionContext() + >>> df = ctx.from_pydict({"a": ["hello"]}) + >>> result = df.select(dfn.functions.upper(dfn.col("a")).alias("u")) + >>> result.collect_column("u")[0].as_py() + 'HELLO' """ return Expr(f.upper(arg.expr)) @@ -2207,13 +2324,14 @@ def make_array(*args: Expr) -> Expr: """Returns an array using the specified input expressions. Examples: - --------- - >>> ctx = dfn.SessionContext() - >>> df = ctx.from_pydict({"a": [1]}) - >>> result = df.select( - ... dfn.functions.make_array(dfn.lit(1), dfn.lit(2), dfn.lit(3)).alias("arr")) - >>> result.collect_column("arr")[0].as_py() - [1, 2, 3] + >>> ctx = dfn.SessionContext() + >>> df = ctx.from_pydict({"a": [1]}) + >>> result = df.select( + ... dfn.functions.make_array( + ... dfn.lit(1), dfn.lit(2), dfn.lit(3) + ... ).alias("arr")) + >>> result.collect_column("arr")[0].as_py() + [1, 2, 3] """ args = [arg.expr for arg in args] return Expr(f.make_array(args)) @@ -2222,7 +2340,8 @@ def make_array(*args: Expr) -> Expr: def make_list(*args: Expr) -> Expr: """Returns an array using the specified input expressions. - This is an alias for :py:func:`make_array`. + See Also: + This is an alias for :py:func:`make_array`. """ return make_array(*args) @@ -2230,7 +2349,8 @@ def make_list(*args: Expr) -> Expr: def array(*args: Expr) -> Expr: """Returns an array using the specified input expressions. - This is an alias for :py:func:`make_array`. + See Also: + This is an alias for :py:func:`make_array`. """ return make_array(*args) @@ -2239,13 +2359,12 @@ def range(start: Expr, stop: Expr, step: Expr) -> Expr: """Create a list of values in the range between start and stop. Examples: - --------- - >>> ctx = dfn.SessionContext() - >>> df = ctx.from_pydict({"a": [1]}) - >>> result = df.select( - ... dfn.functions.range(dfn.lit(0), dfn.lit(5), dfn.lit(2)).alias("r")) - >>> result.collect_column("r")[0].as_py() - [0, 2, 4] + >>> ctx = dfn.SessionContext() + >>> df = ctx.from_pydict({"a": [1]}) + >>> result = df.select( + ... dfn.functions.range(dfn.lit(0), dfn.lit(5), dfn.lit(2)).alias("r")) + >>> result.collect_column("r")[0].as_py() + [0, 2, 4] """ return Expr(f.range(start.expr, stop.expr, step.expr)) @@ -2377,13 +2496,12 @@ def array_append(array: Expr, element: Expr) -> Expr: """Appends an element to the end of an array. Examples: - --------- - >>> ctx = dfn.SessionContext() - >>> df = ctx.from_pydict({"a": [[1, 2, 3]]}) - >>> result = df.select( - ... dfn.functions.array_append(dfn.col("a"), dfn.lit(4)).alias("result")) - >>> result.collect_column("result")[0].as_py() - [1, 2, 3, 4] + >>> ctx = dfn.SessionContext() + >>> df = ctx.from_pydict({"a": [[1, 2, 3]]}) + >>> result = df.select( + ... dfn.functions.array_append(dfn.col("a"), dfn.lit(4)).alias("result")) + >>> result.collect_column("result")[0].as_py() + [1, 2, 3, 4] """ return Expr(f.array_append(array.expr, element.expr)) @@ -2391,7 +2509,8 @@ def array_append(array: Expr, element: Expr) -> Expr: def array_push_back(array: Expr, element: Expr) -> Expr: """Appends an element to the end of an array. - This is an alias for :py:func:`array_append`. + See Also: + This is an alias for :py:func:`array_append`. """ return array_append(array, element) @@ -2399,7 +2518,8 @@ def array_push_back(array: Expr, element: Expr) -> Expr: def list_append(array: Expr, element: Expr) -> Expr: """Appends an element to the end of an array. - This is an alias for :py:func:`array_append`. + See Also: + This is an alias for :py:func:`array_append`. """ return array_append(array, element) @@ -2407,7 +2527,8 @@ def list_append(array: Expr, element: Expr) -> Expr: def list_push_back(array: Expr, element: Expr) -> Expr: """Appends an element to the end of an array. - This is an alias for :py:func:`array_append`. + See Also: + This is an alias for :py:func:`array_append`. """ return array_append(array, element) @@ -2416,13 +2537,12 @@ def array_concat(*args: Expr) -> Expr: """Concatenates the input arrays. Examples: - --------- - >>> ctx = dfn.SessionContext() - >>> df = ctx.from_pydict({"a": [[1, 2]], "b": [[3, 4]]}) - >>> result = df.select( - ... dfn.functions.array_concat(dfn.col("a"), dfn.col("b")).alias("result")) - >>> result.collect_column("result")[0].as_py() - [1, 2, 3, 4] + >>> ctx = dfn.SessionContext() + >>> df = ctx.from_pydict({"a": [[1, 2]], "b": [[3, 4]]}) + >>> result = df.select( + ... dfn.functions.array_concat(dfn.col("a"), dfn.col("b")).alias("result")) + >>> result.collect_column("result")[0].as_py() + [1, 2, 3, 4] """ args = [arg.expr for arg in args] return Expr(f.array_concat(args)) @@ -2431,7 +2551,8 @@ def array_concat(*args: Expr) -> Expr: def array_cat(*args: Expr) -> Expr: """Concatenates the input arrays. - This is an alias for :py:func:`array_concat`. + See Also: + This is an alias for :py:func:`array_concat`. """ return array_concat(*args) @@ -2440,12 +2561,11 @@ def array_dims(array: Expr) -> Expr: """Returns an array of the array's dimensions. Examples: - --------- - >>> ctx = dfn.SessionContext() - >>> df = ctx.from_pydict({"a": [[1, 2, 3]]}) - >>> result = df.select(dfn.functions.array_dims(dfn.col("a")).alias("result")) - >>> result.collect_column("result")[0].as_py() - [3] + >>> ctx = dfn.SessionContext() + >>> df = ctx.from_pydict({"a": [[1, 2, 3]]}) + >>> result = df.select(dfn.functions.array_dims(dfn.col("a")).alias("result")) + >>> result.collect_column("result")[0].as_py() + [3] """ return Expr(f.array_dims(array.expr)) @@ -2454,18 +2574,17 @@ def array_distinct(array: Expr) -> Expr: """Returns distinct values from the array after removing duplicates. Examples: - --------- - >>> ctx = dfn.SessionContext() - >>> df = ctx.from_pydict({"a": [[1, 1, 2, 3]]}) - >>> result = df.select( - ... dfn.functions.array_distinct( - ... dfn.col("a") - ... ).alias("result") - ... ) - >>> sorted( - ... result.collect_column("result")[0].as_py() - ... ) - [1, 2, 3] + >>> ctx = dfn.SessionContext() + >>> df = ctx.from_pydict({"a": [[1, 1, 2, 3]]}) + >>> result = df.select( + ... dfn.functions.array_distinct( + ... dfn.col("a") + ... ).alias("result") + ... ) + >>> sorted( + ... result.collect_column("result")[0].as_py() + ... ) + [1, 2, 3] """ return Expr(f.array_distinct(array.expr)) @@ -2473,7 +2592,8 @@ def array_distinct(array: Expr) -> Expr: def list_cat(*args: Expr) -> Expr: """Concatenates the input arrays. - This is an alias for :py:func:`array_concat`, :py:func:`array_cat`. + See Also: + This is an alias for :py:func:`array_concat`, :py:func:`array_cat`. """ return array_concat(*args) @@ -2481,7 +2601,8 @@ def list_cat(*args: Expr) -> Expr: def list_concat(*args: Expr) -> Expr: """Concatenates the input arrays. - This is an alias for :py:func:`array_concat`, :py:func:`array_cat`. + See Also: + This is an alias for :py:func:`array_concat`, :py:func:`array_cat`. """ return array_concat(*args) @@ -2489,7 +2610,8 @@ def list_concat(*args: Expr) -> Expr: def list_distinct(array: Expr) -> Expr: """Returns distinct values from the array after removing duplicates. - This is an alias for :py:func:`array_distinct`. + See Also: + This is an alias for :py:func:`array_distinct`. """ return array_distinct(array) @@ -2497,7 +2619,8 @@ def list_distinct(array: Expr) -> Expr: def list_dims(array: Expr) -> Expr: """Returns an array of the array's dimensions. - This is an alias for :py:func:`array_dims`. + See Also: + This is an alias for :py:func:`array_dims`. """ return array_dims(array) @@ -2506,13 +2629,12 @@ def array_element(array: Expr, n: Expr) -> Expr: """Extracts the element with the index n from the array. Examples: - --------- - >>> ctx = dfn.SessionContext() - >>> df = ctx.from_pydict({"a": [[10, 20, 30]]}) - >>> result = df.select( - ... dfn.functions.array_element(dfn.col("a"), dfn.lit(2)).alias("result")) - >>> result.collect_column("result")[0].as_py() - 20 + >>> ctx = dfn.SessionContext() + >>> df = ctx.from_pydict({"a": [[10, 20, 30]]}) + >>> result = df.select( + ... dfn.functions.array_element(dfn.col("a"), dfn.lit(2)).alias("result")) + >>> result.collect_column("result")[0].as_py() + 20 """ return Expr(f.array_element(array.expr, n.expr)) @@ -2521,12 +2643,11 @@ def array_empty(array: Expr) -> Expr: """Returns a boolean indicating whether the array is empty. Examples: - --------- - >>> ctx = dfn.SessionContext() - >>> df = ctx.from_pydict({"a": [[1, 2]]}) - >>> result = df.select(dfn.functions.array_empty(dfn.col("a")).alias("result")) - >>> result.collect_column("result")[0].as_py() - False + >>> ctx = dfn.SessionContext() + >>> df = ctx.from_pydict({"a": [[1, 2]]}) + >>> result = df.select(dfn.functions.array_empty(dfn.col("a")).alias("result")) + >>> result.collect_column("result")[0].as_py() + False """ return Expr(f.array_empty(array.expr)) @@ -2534,7 +2655,8 @@ def array_empty(array: Expr) -> Expr: def array_extract(array: Expr, n: Expr) -> Expr: """Extracts the element with the index n from the array. - This is an alias for :py:func:`array_element`. + See Also: + This is an alias for :py:func:`array_element`. """ return array_element(array, n) @@ -2542,7 +2664,8 @@ def array_extract(array: Expr, n: Expr) -> Expr: def list_element(array: Expr, n: Expr) -> Expr: """Extracts the element with the index n from the array. - This is an alias for :py:func:`array_element`. + See Also: + This is an alias for :py:func:`array_element`. """ return array_element(array, n) @@ -2550,7 +2673,8 @@ def list_element(array: Expr, n: Expr) -> Expr: def list_extract(array: Expr, n: Expr) -> Expr: """Extracts the element with the index n from the array. - This is an alias for :py:func:`array_element`. + See Also: + This is an alias for :py:func:`array_element`. """ return array_element(array, n) @@ -2559,12 +2683,11 @@ def array_length(array: Expr) -> Expr: """Returns the length of the array. Examples: - --------- - >>> ctx = dfn.SessionContext() - >>> df = ctx.from_pydict({"a": [[1, 2, 3]]}) - >>> result = df.select(dfn.functions.array_length(dfn.col("a")).alias("result")) - >>> result.collect_column("result")[0].as_py() - 3 + >>> ctx = dfn.SessionContext() + >>> df = ctx.from_pydict({"a": [[1, 2, 3]]}) + >>> result = df.select(dfn.functions.array_length(dfn.col("a")).alias("result")) + >>> result.collect_column("result")[0].as_py() + 3 """ return Expr(f.array_length(array.expr)) @@ -2572,7 +2695,8 @@ def array_length(array: Expr) -> Expr: def list_length(array: Expr) -> Expr: """Returns the length of the array. - This is an alias for :py:func:`array_length`. + See Also: + This is an alias for :py:func:`array_length`. """ return array_length(array) @@ -2581,13 +2705,12 @@ def array_has(first_array: Expr, second_array: Expr) -> Expr: """Returns true if the element appears in the first array, otherwise false. Examples: - --------- - >>> ctx = dfn.SessionContext() - >>> df = ctx.from_pydict({"a": [[1, 2, 3]]}) - >>> result = df.select( - ... dfn.functions.array_has(dfn.col("a"), dfn.lit(2)).alias("result")) - >>> result.collect_column("result")[0].as_py() - True + >>> ctx = dfn.SessionContext() + >>> df = ctx.from_pydict({"a": [[1, 2, 3]]}) + >>> result = df.select( + ... dfn.functions.array_has(dfn.col("a"), dfn.lit(2)).alias("result")) + >>> result.collect_column("result")[0].as_py() + True """ return Expr(f.array_has(first_array.expr, second_array.expr)) @@ -2599,13 +2722,12 @@ def array_has_all(first_array: Expr, second_array: Expr) -> Expr: Otherwise, it returns false. Examples: - --------- - >>> ctx = dfn.SessionContext() - >>> df = ctx.from_pydict({"a": [[1, 2, 3]], "b": [[1, 2]]}) - >>> result = df.select( - ... dfn.functions.array_has_all(dfn.col("a"), dfn.col("b")).alias("result")) - >>> result.collect_column("result")[0].as_py() - True + >>> ctx = dfn.SessionContext() + >>> df = ctx.from_pydict({"a": [[1, 2, 3]], "b": [[1, 2]]}) + >>> result = df.select( + ... dfn.functions.array_has_all(dfn.col("a"), dfn.col("b")).alias("result")) + >>> result.collect_column("result")[0].as_py() + True """ return Expr(f.array_has_all(first_array.expr, second_array.expr)) @@ -2617,13 +2739,12 @@ def array_has_any(first_array: Expr, second_array: Expr) -> Expr: array. Otherwise, it returns false. Examples: - --------- - >>> ctx = dfn.SessionContext() - >>> df = ctx.from_pydict({"a": [[1, 2, 3]], "b": [[2, 5]]}) - >>> result = df.select( - ... dfn.functions.array_has_any(dfn.col("a"), dfn.col("b")).alias("result")) - >>> result.collect_column("result")[0].as_py() - True + >>> ctx = dfn.SessionContext() + >>> df = ctx.from_pydict({"a": [[1, 2, 3]], "b": [[2, 5]]}) + >>> result = df.select( + ... dfn.functions.array_has_any(dfn.col("a"), dfn.col("b")).alias("result")) + >>> result.collect_column("result")[0].as_py() + True """ return Expr(f.array_has_any(first_array.expr, second_array.expr)) @@ -2632,13 +2753,22 @@ def array_position(array: Expr, element: Expr, index: int | None = 1) -> Expr: """Return the position of the first occurrence of ``element`` in ``array``. Examples: - --------- - >>> ctx = dfn.SessionContext() - >>> df = ctx.from_pydict({"a": [[10, 20, 30]]}) - >>> result = df.select( - ... dfn.functions.array_position(dfn.col("a"), dfn.lit(20)).alias("result")) - >>> result.collect_column("result")[0].as_py() - 2 + >>> ctx = dfn.SessionContext() + >>> df = ctx.from_pydict({"a": [[10, 20, 30]]}) + >>> result = df.select( + ... dfn.functions.array_position(dfn.col("a"), dfn.lit(20)).alias("result")) + >>> result.collect_column("result")[0].as_py() + 2 + + Use ``index`` to start searching from a given position: + + >>> df = ctx.from_pydict({"a": [[10, 20, 10, 20]]}) + >>> result = df.select( + ... dfn.functions.array_position( + ... dfn.col("a"), dfn.lit(20), index=3, + ... ).alias("result")) + >>> result.collect_column("result")[0].as_py() + 4 """ return Expr(f.array_position(array.expr, element.expr, index)) @@ -2646,7 +2776,8 @@ def array_position(array: Expr, element: Expr, index: int | None = 1) -> Expr: def array_indexof(array: Expr, element: Expr, index: int | None = 1) -> Expr: """Return the position of the first occurrence of ``element`` in ``array``. - This is an alias for :py:func:`array_position`. + See Also: + This is an alias for :py:func:`array_position`. """ return array_position(array, element, index) @@ -2654,7 +2785,8 @@ def array_indexof(array: Expr, element: Expr, index: int | None = 1) -> Expr: def list_position(array: Expr, element: Expr, index: int | None = 1) -> Expr: """Return the position of the first occurrence of ``element`` in ``array``. - This is an alias for :py:func:`array_position`. + See Also: + This is an alias for :py:func:`array_position`. """ return array_position(array, element, index) @@ -2662,7 +2794,8 @@ def list_position(array: Expr, element: Expr, index: int | None = 1) -> Expr: def list_indexof(array: Expr, element: Expr, index: int | None = 1) -> Expr: """Return the position of the first occurrence of ``element`` in ``array``. - This is an alias for :py:func:`array_position`. + See Also: + This is an alias for :py:func:`array_position`. """ return array_position(array, element, index) @@ -2671,13 +2804,12 @@ def array_positions(array: Expr, element: Expr) -> Expr: """Searches for an element in the array and returns all occurrences. Examples: - --------- - >>> ctx = dfn.SessionContext() - >>> df = ctx.from_pydict({"a": [[1, 2, 1]]}) - >>> result = df.select( - ... dfn.functions.array_positions(dfn.col("a"), dfn.lit(1)).alias("result")) - >>> result.collect_column("result")[0].as_py() - [1, 3] + >>> ctx = dfn.SessionContext() + >>> df = ctx.from_pydict({"a": [[1, 2, 1]]}) + >>> result = df.select( + ... dfn.functions.array_positions(dfn.col("a"), dfn.lit(1)).alias("result")) + >>> result.collect_column("result")[0].as_py() + [1, 3] """ return Expr(f.array_positions(array.expr, element.expr)) @@ -2685,7 +2817,8 @@ def array_positions(array: Expr, element: Expr) -> Expr: def list_positions(array: Expr, element: Expr) -> Expr: """Searches for an element in the array and returns all occurrences. - This is an alias for :py:func:`array_positions`. + See Also: + This is an alias for :py:func:`array_positions`. """ return array_positions(array, element) @@ -2694,12 +2827,11 @@ def array_ndims(array: Expr) -> Expr: """Returns the number of dimensions of the array. Examples: - --------- - >>> ctx = dfn.SessionContext() - >>> df = ctx.from_pydict({"a": [[1, 2, 3]]}) - >>> result = df.select(dfn.functions.array_ndims(dfn.col("a")).alias("result")) - >>> result.collect_column("result")[0].as_py() - 1 + >>> ctx = dfn.SessionContext() + >>> df = ctx.from_pydict({"a": [[1, 2, 3]]}) + >>> result = df.select(dfn.functions.array_ndims(dfn.col("a")).alias("result")) + >>> result.collect_column("result")[0].as_py() + 1 """ return Expr(f.array_ndims(array.expr)) @@ -2707,7 +2839,8 @@ def array_ndims(array: Expr) -> Expr: def list_ndims(array: Expr) -> Expr: """Returns the number of dimensions of the array. - This is an alias for :py:func:`array_ndims`. + See Also: + This is an alias for :py:func:`array_ndims`. """ return array_ndims(array) @@ -2716,13 +2849,12 @@ def array_prepend(element: Expr, array: Expr) -> Expr: """Prepends an element to the beginning of an array. Examples: - --------- - >>> ctx = dfn.SessionContext() - >>> df = ctx.from_pydict({"a": [[1, 2]]}) - >>> result = df.select( - ... dfn.functions.array_prepend(dfn.lit(0), dfn.col("a")).alias("result")) - >>> result.collect_column("result")[0].as_py() - [0, 1, 2] + >>> ctx = dfn.SessionContext() + >>> df = ctx.from_pydict({"a": [[1, 2]]}) + >>> result = df.select( + ... dfn.functions.array_prepend(dfn.lit(0), dfn.col("a")).alias("result")) + >>> result.collect_column("result")[0].as_py() + [0, 1, 2] """ return Expr(f.array_prepend(element.expr, array.expr)) @@ -2730,7 +2862,8 @@ def array_prepend(element: Expr, array: Expr) -> Expr: def array_push_front(element: Expr, array: Expr) -> Expr: """Prepends an element to the beginning of an array. - This is an alias for :py:func:`array_prepend`. + See Also: + This is an alias for :py:func:`array_prepend`. """ return array_prepend(element, array) @@ -2738,7 +2871,8 @@ def array_push_front(element: Expr, array: Expr) -> Expr: def list_prepend(element: Expr, array: Expr) -> Expr: """Prepends an element to the beginning of an array. - This is an alias for :py:func:`array_prepend`. + See Also: + This is an alias for :py:func:`array_prepend`. """ return array_prepend(element, array) @@ -2746,7 +2880,8 @@ def list_prepend(element: Expr, array: Expr) -> Expr: def list_push_front(element: Expr, array: Expr) -> Expr: """Prepends an element to the beginning of an array. - This is an alias for :py:func:`array_prepend`. + See Also: + This is an alias for :py:func:`array_prepend`. """ return array_prepend(element, array) @@ -2755,12 +2890,12 @@ def array_pop_back(array: Expr) -> Expr: """Returns the array without the last element. Examples: - --------- - >>> ctx = dfn.SessionContext() - >>> df = ctx.from_pydict({"a": [[1, 2, 3]]}) - >>> result = df.select(dfn.functions.array_pop_back(dfn.col("a")).alias("result")) - >>> result.collect_column("result")[0].as_py() - [1, 2] + >>> ctx = dfn.SessionContext() + >>> df = ctx.from_pydict({"a": [[1, 2, 3]]}) + >>> result = df.select( + ... dfn.functions.array_pop_back(dfn.col("a")).alias("result")) + >>> result.collect_column("result")[0].as_py() + [1, 2] """ return Expr(f.array_pop_back(array.expr)) @@ -2769,12 +2904,12 @@ def array_pop_front(array: Expr) -> Expr: """Returns the array without the first element. Examples: - --------- - >>> ctx = dfn.SessionContext() - >>> df = ctx.from_pydict({"a": [[1, 2, 3]]}) - >>> result = df.select(dfn.functions.array_pop_front(dfn.col("a")).alias("result")) - >>> result.collect_column("result")[0].as_py() - [2, 3] + >>> ctx = dfn.SessionContext() + >>> df = ctx.from_pydict({"a": [[1, 2, 3]]}) + >>> result = df.select( + ... dfn.functions.array_pop_front(dfn.col("a")).alias("result")) + >>> result.collect_column("result")[0].as_py() + [2, 3] """ return Expr(f.array_pop_front(array.expr)) @@ -2783,13 +2918,12 @@ def array_remove(array: Expr, element: Expr) -> Expr: """Removes the first element from the array equal to the given value. Examples: - --------- - >>> ctx = dfn.SessionContext() - >>> df = ctx.from_pydict({"a": [[1, 2, 1]]}) - >>> result = df.select( - ... dfn.functions.array_remove(dfn.col("a"), dfn.lit(1)).alias("result")) - >>> result.collect_column("result")[0].as_py() - [2, 1] + >>> ctx = dfn.SessionContext() + >>> df = ctx.from_pydict({"a": [[1, 2, 1]]}) + >>> result = df.select( + ... dfn.functions.array_remove(dfn.col("a"), dfn.lit(1)).alias("result")) + >>> result.collect_column("result")[0].as_py() + [2, 1] """ return Expr(f.array_remove(array.expr, element.expr)) @@ -2797,7 +2931,8 @@ def array_remove(array: Expr, element: Expr) -> Expr: def list_remove(array: Expr, element: Expr) -> Expr: """Removes the first element from the array equal to the given value. - This is an alias for :py:func:`array_remove`. + See Also: + This is an alias for :py:func:`array_remove`. """ return array_remove(array, element) @@ -2806,14 +2941,13 @@ def array_remove_n(array: Expr, element: Expr, max: Expr) -> Expr: """Removes the first ``max`` elements from the array equal to the given value. Examples: - --------- - >>> ctx = dfn.SessionContext() - >>> df = ctx.from_pydict({"a": [[1, 2, 1, 1]]}) - >>> result = df.select( - ... dfn.functions.array_remove_n(dfn.col("a"), dfn.lit(1), - ... dfn.lit(2)).alias("result")) - >>> result.collect_column("result")[0].as_py() - [2, 1] + >>> ctx = dfn.SessionContext() + >>> df = ctx.from_pydict({"a": [[1, 2, 1, 1]]}) + >>> result = df.select( + ... dfn.functions.array_remove_n(dfn.col("a"), dfn.lit(1), + ... dfn.lit(2)).alias("result")) + >>> result.collect_column("result")[0].as_py() + [2, 1] """ return Expr(f.array_remove_n(array.expr, element.expr, max.expr)) @@ -2821,7 +2955,8 @@ def array_remove_n(array: Expr, element: Expr, max: Expr) -> Expr: def list_remove_n(array: Expr, element: Expr, max: Expr) -> Expr: """Removes the first ``max`` elements from the array equal to the given value. - This is an alias for :py:func:`array_remove_n`. + See Also: + This is an alias for :py:func:`array_remove_n`. """ return array_remove_n(array, element, max) @@ -2830,13 +2965,14 @@ def array_remove_all(array: Expr, element: Expr) -> Expr: """Removes all elements from the array equal to the given value. Examples: - --------- - >>> ctx = dfn.SessionContext() - >>> df = ctx.from_pydict({"a": [[1, 2, 1]]}) - >>> result = df.select( - ... dfn.functions.array_remove_all(dfn.col("a"), dfn.lit(1)).alias("result")) - >>> result.collect_column("result")[0].as_py() - [2] + >>> ctx = dfn.SessionContext() + >>> df = ctx.from_pydict({"a": [[1, 2, 1]]}) + >>> result = df.select( + ... dfn.functions.array_remove_all( + ... dfn.col("a"), dfn.lit(1) + ... ).alias("result")) + >>> result.collect_column("result")[0].as_py() + [2] """ return Expr(f.array_remove_all(array.expr, element.expr)) @@ -2844,7 +2980,8 @@ def array_remove_all(array: Expr, element: Expr) -> Expr: def list_remove_all(array: Expr, element: Expr) -> Expr: """Removes all elements from the array equal to the given value. - This is an alias for :py:func:`array_remove_all`. + See Also: + This is an alias for :py:func:`array_remove_all`. """ return array_remove_all(array, element) @@ -2853,13 +2990,12 @@ def array_repeat(element: Expr, count: Expr) -> Expr: """Returns an array containing ``element`` ``count`` times. Examples: - --------- - >>> ctx = dfn.SessionContext() - >>> df = ctx.from_pydict({"a": [1]}) - >>> result = df.select( - ... dfn.functions.array_repeat(dfn.lit(3), dfn.lit(3)).alias("result")) - >>> result.collect_column("result")[0].as_py() - [3, 3, 3] + >>> ctx = dfn.SessionContext() + >>> df = ctx.from_pydict({"a": [1]}) + >>> result = df.select( + ... dfn.functions.array_repeat(dfn.lit(3), dfn.lit(3)).alias("result")) + >>> result.collect_column("result")[0].as_py() + [3, 3, 3] """ return Expr(f.array_repeat(element.expr, count.expr)) @@ -2867,7 +3003,8 @@ def array_repeat(element: Expr, count: Expr) -> Expr: def list_repeat(element: Expr, count: Expr) -> Expr: """Returns an array containing ``element`` ``count`` times. - This is an alias for :py:func:`array_repeat`. + See Also: + This is an alias for :py:func:`array_repeat`. """ return array_repeat(element, count) @@ -2876,14 +3013,13 @@ def array_replace(array: Expr, from_val: Expr, to_val: Expr) -> Expr: """Replaces the first occurrence of ``from_val`` with ``to_val``. Examples: - --------- - >>> ctx = dfn.SessionContext() - >>> df = ctx.from_pydict({"a": [[1, 2, 1]]}) - >>> result = df.select( - ... dfn.functions.array_replace(dfn.col("a"), dfn.lit(1), - ... dfn.lit(9)).alias("result")) - >>> result.collect_column("result")[0].as_py() - [9, 2, 1] + >>> ctx = dfn.SessionContext() + >>> df = ctx.from_pydict({"a": [[1, 2, 1]]}) + >>> result = df.select( + ... dfn.functions.array_replace(dfn.col("a"), dfn.lit(1), + ... dfn.lit(9)).alias("result")) + >>> result.collect_column("result")[0].as_py() + [9, 2, 1] """ return Expr(f.array_replace(array.expr, from_val.expr, to_val.expr)) @@ -2891,7 +3027,8 @@ def array_replace(array: Expr, from_val: Expr, to_val: Expr) -> Expr: def list_replace(array: Expr, from_val: Expr, to_val: Expr) -> Expr: """Replaces the first occurrence of ``from_val`` with ``to_val``. - This is an alias for :py:func:`array_replace`. + See Also: + This is an alias for :py:func:`array_replace`. """ return array_replace(array, from_val, to_val) @@ -2903,14 +3040,13 @@ def array_replace_n(array: Expr, from_val: Expr, to_val: Expr, max: Expr) -> Exp specified element. Examples: - --------- - >>> ctx = dfn.SessionContext() - >>> df = ctx.from_pydict({"a": [[1, 2, 1, 1]]}) - >>> result = df.select( - ... dfn.functions.array_replace_n(dfn.col("a"), dfn.lit(1), dfn.lit(9), - ... dfn.lit(2)).alias("result")) - >>> result.collect_column("result")[0].as_py() - [9, 2, 9, 1] + >>> ctx = dfn.SessionContext() + >>> df = ctx.from_pydict({"a": [[1, 2, 1, 1]]}) + >>> result = df.select( + ... dfn.functions.array_replace_n(dfn.col("a"), dfn.lit(1), dfn.lit(9), + ... dfn.lit(2)).alias("result")) + >>> result.collect_column("result")[0].as_py() + [9, 2, 9, 1] """ return Expr(f.array_replace_n(array.expr, from_val.expr, to_val.expr, max.expr)) @@ -2921,7 +3057,8 @@ def list_replace_n(array: Expr, from_val: Expr, to_val: Expr, max: Expr) -> Expr Replaces the first ``max`` occurrences of the specified element with another specified element. - This is an alias for :py:func:`array_replace_n`. + See Also: + This is an alias for :py:func:`array_replace_n`. """ return array_replace_n(array, from_val, to_val, max) @@ -2930,14 +3067,13 @@ def array_replace_all(array: Expr, from_val: Expr, to_val: Expr) -> Expr: """Replaces all occurrences of ``from_val`` with ``to_val``. Examples: - --------- - >>> ctx = dfn.SessionContext() - >>> df = ctx.from_pydict({"a": [[1, 2, 1]]}) - >>> result = df.select( - ... dfn.functions.array_replace_all(dfn.col("a"), dfn.lit(1), - ... dfn.lit(9)).alias("result")) - >>> result.collect_column("result")[0].as_py() - [9, 2, 9] + >>> ctx = dfn.SessionContext() + >>> df = ctx.from_pydict({"a": [[1, 2, 1]]}) + >>> result = df.select( + ... dfn.functions.array_replace_all(dfn.col("a"), dfn.lit(1), + ... dfn.lit(9)).alias("result")) + >>> result.collect_column("result")[0].as_py() + [9, 2, 9] """ return Expr(f.array_replace_all(array.expr, from_val.expr, to_val.expr)) @@ -2945,7 +3081,8 @@ def array_replace_all(array: Expr, from_val: Expr, to_val: Expr) -> Expr: def list_replace_all(array: Expr, from_val: Expr, to_val: Expr) -> Expr: """Replaces all occurrences of ``from_val`` with ``to_val``. - This is an alias for :py:func:`array_replace_all`. + See Also: + This is an alias for :py:func:`array_replace_all`. """ return array_replace_all(array, from_val, to_val) @@ -2959,12 +3096,19 @@ def array_sort(array: Expr, descending: bool = False, null_first: bool = False) null_first: If True, nulls will be returned at the beginning of the array. Examples: - --------- - >>> ctx = dfn.SessionContext() - >>> df = ctx.from_pydict({"a": [[3, 1, 2]]}) - >>> result = df.select(dfn.functions.array_sort(dfn.col("a")).alias("result")) - >>> result.collect_column("result")[0].as_py() - [1, 2, 3] + >>> ctx = dfn.SessionContext() + >>> df = ctx.from_pydict({"a": [[3, 1, 2]]}) + >>> result = df.select(dfn.functions.array_sort(dfn.col("a")).alias("result")) + >>> result.collect_column("result")[0].as_py() + [1, 2, 3] + + >>> df = ctx.from_pydict({"a": [[3, None, 1]]}) + >>> result = df.select( + ... dfn.functions.array_sort( + ... dfn.col("a"), descending=True, null_first=True, + ... ).alias("result")) + >>> result.collect_column("result")[0].as_py() + [None, 3, 1] """ desc = "DESC" if descending else "ASC" nulls_first = "NULLS FIRST" if null_first else "NULLS LAST" @@ -2978,7 +3122,11 @@ def array_sort(array: Expr, descending: bool = False, null_first: bool = False) def list_sort(array: Expr, descending: bool = False, null_first: bool = False) -> Expr: - """This is an alias for :py:func:`array_sort`.""" + """Sorts the array. + + See Also: + This is an alias for :py:func:`array_sort`. + """ return array_sort(array, descending=descending, null_first=null_first) @@ -2988,14 +3136,23 @@ def array_slice( """Returns a slice of the array. Examples: - --------- - >>> ctx = dfn.SessionContext() - >>> df = ctx.from_pydict({"a": [[1, 2, 3, 4]]}) - >>> result = df.select( - ... dfn.functions.array_slice(dfn.col("a"), dfn.lit(2), - ... dfn.lit(3)).alias("result")) - >>> result.collect_column("result")[0].as_py() - [2, 3] + >>> ctx = dfn.SessionContext() + >>> df = ctx.from_pydict({"a": [[1, 2, 3, 4]]}) + >>> result = df.select( + ... dfn.functions.array_slice(dfn.col("a"), dfn.lit(2), + ... dfn.lit(3)).alias("result")) + >>> result.collect_column("result")[0].as_py() + [2, 3] + + Use ``stride`` to skip elements: + + >>> result = df.select( + ... dfn.functions.array_slice( + ... dfn.col("a"), dfn.lit(1), dfn.lit(4), + ... stride=dfn.lit(2), + ... ).alias("result")) + >>> result.collect_column("result")[0].as_py() + [1, 3] """ if stride is not None: stride = stride.expr @@ -3005,7 +3162,8 @@ def array_slice( def list_slice(array: Expr, begin: Expr, end: Expr, stride: Expr | None = None) -> Expr: """Returns a slice of the array. - This is an alias for :py:func:`array_slice`. + See Also: + This is an alias for :py:func:`array_slice`. """ return array_slice(array, begin, end, stride) @@ -3014,18 +3172,17 @@ def array_intersect(array1: Expr, array2: Expr) -> Expr: """Returns the intersection of ``array1`` and ``array2``. Examples: - --------- - >>> ctx = dfn.SessionContext() - >>> df = ctx.from_pydict({"a": [[1, 2, 3]], "b": [[2, 3, 4]]}) - >>> result = df.select( - ... dfn.functions.array_intersect( - ... dfn.col("a"), dfn.col("b") - ... ).alias("result") - ... ) - >>> sorted( - ... result.collect_column("result")[0].as_py() - ... ) - [2, 3] + >>> ctx = dfn.SessionContext() + >>> df = ctx.from_pydict({"a": [[1, 2, 3]], "b": [[2, 3, 4]]}) + >>> result = df.select( + ... dfn.functions.array_intersect( + ... dfn.col("a"), dfn.col("b") + ... ).alias("result") + ... ) + >>> sorted( + ... result.collect_column("result")[0].as_py() + ... ) + [2, 3] """ return Expr(f.array_intersect(array1.expr, array2.expr)) @@ -3033,7 +3190,8 @@ def array_intersect(array1: Expr, array2: Expr) -> Expr: def list_intersect(array1: Expr, array2: Expr) -> Expr: """Returns an the intersection of ``array1`` and ``array2``. - This is an alias for :py:func:`array_intersect`. + See Also: + This is an alias for :py:func:`array_intersect`. """ return array_intersect(array1, array2) @@ -3044,18 +3202,17 @@ def array_union(array1: Expr, array2: Expr) -> Expr: Duplicate rows will not be returned. Examples: - --------- - >>> ctx = dfn.SessionContext() - >>> df = ctx.from_pydict({"a": [[1, 2, 3]], "b": [[2, 3, 4]]}) - >>> result = df.select( - ... dfn.functions.array_union( - ... dfn.col("a"), dfn.col("b") - ... ).alias("result") - ... ) - >>> sorted( - ... result.collect_column("result")[0].as_py() - ... ) - [1, 2, 3, 4] + >>> ctx = dfn.SessionContext() + >>> df = ctx.from_pydict({"a": [[1, 2, 3]], "b": [[2, 3, 4]]}) + >>> result = df.select( + ... dfn.functions.array_union( + ... dfn.col("a"), dfn.col("b") + ... ).alias("result") + ... ) + >>> sorted( + ... result.collect_column("result")[0].as_py() + ... ) + [1, 2, 3, 4] """ return Expr(f.array_union(array1.expr, array2.expr)) @@ -3065,7 +3222,8 @@ def list_union(array1: Expr, array2: Expr) -> Expr: Duplicate rows will not be returned. - This is an alias for :py:func:`array_union`. + See Also: + This is an alias for :py:func:`array_union`. """ return array_union(array1, array2) @@ -3074,13 +3232,12 @@ def array_except(array1: Expr, array2: Expr) -> Expr: """Returns the elements that appear in ``array1`` but not in ``array2``. Examples: - --------- - >>> ctx = dfn.SessionContext() - >>> df = ctx.from_pydict({"a": [[1, 2, 3]], "b": [[2, 3, 4]]}) - >>> result = df.select( - ... dfn.functions.array_except(dfn.col("a"), dfn.col("b")).alias("result")) - >>> result.collect_column("result")[0].as_py() - [1] + >>> ctx = dfn.SessionContext() + >>> df = ctx.from_pydict({"a": [[1, 2, 3]], "b": [[2, 3, 4]]}) + >>> result = df.select( + ... dfn.functions.array_except(dfn.col("a"), dfn.col("b")).alias("result")) + >>> result.collect_column("result")[0].as_py() + [1] """ return Expr(f.array_except(array1.expr, array2.expr)) @@ -3088,7 +3245,8 @@ def array_except(array1: Expr, array2: Expr) -> Expr: def list_except(array1: Expr, array2: Expr) -> Expr: """Returns the elements that appear in ``array1`` but not in the ``array2``. - This is an alias for :py:func:`array_except`. + See Also: + This is an alias for :py:func:`array_except`. """ return array_except(array1, array2) @@ -3100,14 +3258,13 @@ def array_resize(array: Expr, size: Expr, value: Expr) -> Expr: be filled with the given ``value``. Examples: - --------- - >>> ctx = dfn.SessionContext() - >>> df = ctx.from_pydict({"a": [[1, 2]]}) - >>> result = df.select( - ... dfn.functions.array_resize(dfn.col("a"), dfn.lit(4), - ... dfn.lit(0)).alias("result")) - >>> result.collect_column("result")[0].as_py() - [1, 2, 0, 0] + >>> ctx = dfn.SessionContext() + >>> df = ctx.from_pydict({"a": [[1, 2]]}) + >>> result = df.select( + ... dfn.functions.array_resize(dfn.col("a"), dfn.lit(4), + ... dfn.lit(0)).alias("result")) + >>> result.collect_column("result")[0].as_py() + [1, 2, 0, 0] """ return Expr(f.array_resize(array.expr, size.expr, value.expr)) @@ -3116,7 +3273,10 @@ def list_resize(array: Expr, size: Expr, value: Expr) -> Expr: """Returns an array with the specified size filled. If ``size`` is greater than the ``array`` length, the additional entries will be - filled with the given ``value``. This is an alias for :py:func:`array_resize`. + filled with the given ``value``. + + See Also: + This is an alias for :py:func:`array_resize`. """ return array_resize(array, size, value) @@ -3125,12 +3285,11 @@ def flatten(array: Expr) -> Expr: """Flattens an array of arrays into a single array. Examples: - --------- - >>> ctx = dfn.SessionContext() - >>> df = ctx.from_pydict({"a": [[[1, 2], [3, 4]]]}) - >>> result = df.select(dfn.functions.flatten(dfn.col("a")).alias("result")) - >>> result.collect_column("result")[0].as_py() - [1, 2, 3, 4] + >>> ctx = dfn.SessionContext() + >>> df = ctx.from_pydict({"a": [[[1, 2], [3, 4]]]}) + >>> result = df.select(dfn.functions.flatten(dfn.col("a")).alias("result")) + >>> result.collect_column("result")[0].as_py() + [1, 2, 3, 4] """ return Expr(f.flatten(array.expr)) @@ -3139,18 +3298,21 @@ def cardinality(array: Expr) -> Expr: """Returns the total number of elements in the array. Examples: - --------- - >>> ctx = dfn.SessionContext() - >>> df = ctx.from_pydict({"a": [[1, 2, 3]]}) - >>> result = df.select(dfn.functions.cardinality(dfn.col("a")).alias("result")) - >>> result.collect_column("result")[0].as_py() - 3 + >>> ctx = dfn.SessionContext() + >>> df = ctx.from_pydict({"a": [[1, 2, 3]]}) + >>> result = df.select(dfn.functions.cardinality(dfn.col("a")).alias("result")) + >>> result.collect_column("result")[0].as_py() + 3 """ return Expr(f.cardinality(array.expr)) def empty(array: Expr) -> Expr: - """This is an alias for :py:func:`array_empty`.""" + """Returns true if the array is empty. + + See Also: + This is an alias for :py:func:`array_empty`. + """ return array_empty(array) @@ -3173,13 +3335,20 @@ def approx_distinct( filter: If provided, only compute against rows for which the filter is True Examples: - --------- - >>> ctx = dfn.SessionContext() - >>> df = ctx.from_pydict({"a": [1, 1, 2, 3]}) - >>> result = df.aggregate( - ... [], [dfn.functions.approx_distinct(dfn.col("a")).alias("v")]) - >>> result.collect_column("v")[0].as_py() == 3 - True + >>> ctx = dfn.SessionContext() + >>> df = ctx.from_pydict({"a": [1, 1, 2, 3]}) + >>> result = df.aggregate( + ... [], [dfn.functions.approx_distinct(dfn.col("a")).alias("v")]) + >>> result.collect_column("v")[0].as_py() == 3 + True + + >>> result = df.aggregate( + ... [], [dfn.functions.approx_distinct( + ... dfn.col("a"), + ... filter=dfn.col("a") > dfn.lit(1) + ... ).alias("v")]) + >>> result.collect_column("v")[0].as_py() == 2 + True """ filter_raw = filter.expr if filter is not None else None @@ -3200,13 +3369,20 @@ def approx_median(expression: Expr, filter: Expr | None = None) -> Expr: filter: If provided, only compute against rows for which the filter is True Examples: - --------- - >>> ctx = dfn.SessionContext() - >>> df = ctx.from_pydict({"a": [1.0, 2.0, 3.0]}) - >>> result = df.aggregate( - ... [], [dfn.functions.approx_median(dfn.col("a")).alias("v")]) - >>> result.collect_column("v")[0].as_py() - 2.0 + >>> ctx = dfn.SessionContext() + >>> df = ctx.from_pydict({"a": [1.0, 2.0, 3.0]}) + >>> result = df.aggregate( + ... [], [dfn.functions.approx_median(dfn.col("a")).alias("v")]) + >>> result.collect_column("v")[0].as_py() + 2.0 + + >>> result = df.aggregate( + ... [], [dfn.functions.approx_median( + ... dfn.col("a"), + ... filter=dfn.col("a") > dfn.lit(1.0) + ... ).alias("v")]) + >>> result.collect_column("v")[0].as_py() + 2.5 """ filter_raw = filter.expr if filter is not None else None return Expr(f.approx_median(expression.expr, filter=filter_raw)) @@ -3240,13 +3416,23 @@ def approx_percentile_cont( filter: If provided, only compute against rows for which the filter is True Examples: - --------- - >>> ctx = dfn.SessionContext() - >>> df = ctx.from_pydict({"a": [1.0, 2.0, 3.0, 4.0, 5.0]}) - >>> result = df.aggregate( - ... [], [dfn.functions.approx_percentile_cont(dfn.col("a"), 0.5).alias("v")]) - >>> result.collect_column("v")[0].as_py() - 3.0 + >>> ctx = dfn.SessionContext() + >>> df = ctx.from_pydict({"a": [1.0, 2.0, 3.0, 4.0, 5.0]}) + >>> result = df.aggregate( + ... [], [dfn.functions.approx_percentile_cont( + ... dfn.col("a"), 0.5 + ... ).alias("v")]) + >>> result.collect_column("v")[0].as_py() + 3.0 + + >>> result = df.aggregate( + ... [], [dfn.functions.approx_percentile_cont( + ... dfn.col("a"), 0.5, + ... num_centroids=10, + ... filter=dfn.col("a") > dfn.lit(1.0), + ... ).alias("v")]) + >>> result.collect_column("v")[0].as_py() + 3.5 """ sort_expr_raw = sort_or_default(sort_expression) filter_raw = filter.expr if filter is not None else None @@ -3280,14 +3466,22 @@ def approx_percentile_cont_with_weight( filter: If provided, only compute against rows for which the filter is True Examples: - --------- - >>> ctx = dfn.SessionContext() - >>> df = ctx.from_pydict({"a": [1.0, 2.0, 3.0], "w": [1.0, 1.0, 1.0]}) - >>> result = df.aggregate( - ... [], [dfn.functions.approx_percentile_cont_with_weight(dfn.col("a"), - ... dfn.col("w"), 0.5).alias("v")]) - >>> result.collect_column("v")[0].as_py() - 2.0 + >>> ctx = dfn.SessionContext() + >>> df = ctx.from_pydict({"a": [1.0, 2.0, 3.0], "w": [1.0, 1.0, 1.0]}) + >>> result = df.aggregate( + ... [], [dfn.functions.approx_percentile_cont_with_weight(dfn.col("a"), + ... dfn.col("w"), 0.5).alias("v")]) + >>> result.collect_column("v")[0].as_py() + 2.0 + + >>> result = df.aggregate( + ... [], [dfn.functions.approx_percentile_cont_with_weight( + ... dfn.col("a"), dfn.col("w"), 0.5, + ... num_centroids=10, + ... filter=dfn.col("a") > dfn.lit(1.0), + ... ).alias("v")]) + >>> result.collect_column("v")[0].as_py() + 2.5 """ sort_expr_raw = sort_or_default(sort_expression) filter_raw = filter.expr if filter is not None else None @@ -3323,17 +3517,30 @@ def array_agg( filter: If provided, only compute against rows for which the filter is True order_by: Order the resultant array values. Accepts column names or expressions. - For example:: + Examples: + >>> ctx = dfn.SessionContext() + >>> df = ctx.from_pydict({"a": [1, 2, 3]}) + >>> result = df.aggregate( + ... [], [dfn.functions.array_agg(dfn.col("a")).alias("v")]) + >>> result.collect_column("v")[0].as_py() + [1, 2, 3] - df.aggregate([], array_agg(col("a"), order_by="b")) + >>> df = ctx.from_pydict({"a": [3, 1, 2, 1]}) + >>> result = df.aggregate( + ... [], [dfn.functions.array_agg( + ... dfn.col("a"), distinct=True, + ... ).alias("v")]) + >>> sorted(result.collect_column("v")[0].as_py()) + [1, 2, 3] - Examples: - --------- - >>> ctx = dfn.SessionContext() - >>> df = ctx.from_pydict({"a": [1, 2, 3]}) - >>> result = df.aggregate([], [dfn.functions.array_agg(dfn.col("a")).alias("v")]) - >>> result.collect_column("v")[0].as_py() - [1, 2, 3] + >>> result = df.aggregate( + ... [], [dfn.functions.array_agg( + ... dfn.col("a"), + ... filter=dfn.col("a") > dfn.lit(1), + ... order_by="a", + ... ).alias("v")]) + >>> result.collect_column("v")[0].as_py() + [2, 3] """ order_by_raw = sort_list_to_raw_sort_list(order_by) filter_raw = filter.expr if filter is not None else None @@ -3361,12 +3568,19 @@ def avg( filter: If provided, only compute against rows for which the filter is True Examples: - --------- - >>> ctx = dfn.SessionContext() - >>> df = ctx.from_pydict({"a": [1.0, 2.0, 3.0]}) - >>> result = df.aggregate([], [dfn.functions.avg(dfn.col("a")).alias("v")]) - >>> result.collect_column("v")[0].as_py() - 2.0 + >>> ctx = dfn.SessionContext() + >>> df = ctx.from_pydict({"a": [1.0, 2.0, 3.0]}) + >>> result = df.aggregate([], [dfn.functions.avg(dfn.col("a")).alias("v")]) + >>> result.collect_column("v")[0].as_py() + 2.0 + + >>> result = df.aggregate( + ... [], [dfn.functions.avg( + ... dfn.col("a"), + ... filter=dfn.col("a") > dfn.lit(1.0) + ... ).alias("v")]) + >>> result.collect_column("v")[0].as_py() + 2.5 """ filter_raw = filter.expr if filter is not None else None return Expr(f.avg(expression.expr, filter=filter_raw)) @@ -3392,6 +3606,14 @@ def corr(value_y: Expr, value_x: Expr, filter: Expr | None = None) -> Expr: ... [], [dfn.functions.corr(dfn.col("a"), dfn.col("b")).alias("v")]) >>> result.collect_column("v")[0].as_py() 1.0 + + >>> result = df.aggregate( + ... [], [dfn.functions.corr( + ... dfn.col("a"), dfn.col("b"), + ... filter=dfn.col("a") > dfn.lit(1.0) + ... ).alias("v")]) + >>> result.collect_column("v")[0].as_py() + 1.0 """ filter_raw = filter.expr if filter is not None else None return Expr(f.corr(value_y.expr, value_x.expr, filter=filter_raw)) @@ -3415,12 +3637,20 @@ def count( filter: If provided, only compute against rows for which the filter is True Examples: - --------- - >>> ctx = dfn.SessionContext() - >>> df = ctx.from_pydict({"a": [1, 2, 3]}) - >>> result = df.aggregate([], [dfn.functions.count(dfn.col("a")).alias("v")]) - >>> result.collect_column("v")[0].as_py() - 3 + >>> ctx = dfn.SessionContext() + >>> df = ctx.from_pydict({"a": [1, 2, 3]}) + >>> result = df.aggregate([], [dfn.functions.count(dfn.col("a")).alias("v")]) + >>> result.collect_column("v")[0].as_py() + 3 + + >>> df = ctx.from_pydict({"a": [1, 1, 2, 3]}) + >>> result = df.aggregate( + ... [], [dfn.functions.count( + ... dfn.col("a"), distinct=True, + ... filter=dfn.col("a") > dfn.lit(1), + ... ).alias("v")]) + >>> result.collect_column("v")[0].as_py() + 2 """ filter_raw = filter.expr if filter is not None else None @@ -3458,6 +3688,18 @@ def covar_pop(value_y: Expr, value_x: Expr, filter: Expr | None = None) -> Expr: ... ) >>> result.collect_column("v")[0].as_py() 3.0 + + >>> df = ctx.from_pydict( + ... {"a": [0.0, 1.0, 3.0], "b": [0.0, 1.0, 3.0]}) + >>> result = df.aggregate( + ... [], + ... [dfn.functions.covar_pop( + ... dfn.col("a"), dfn.col("b"), + ... filter=dfn.col("a") > dfn.lit(0.0) + ... ).alias("v")] + ... ) + >>> result.collect_column("v")[0].as_py() + 1.0 """ filter_raw = filter.expr if filter is not None else None return Expr(f.covar_pop(value_y.expr, value_x.expr, filter=filter_raw)) @@ -3483,6 +3725,14 @@ def covar_samp(value_y: Expr, value_x: Expr, filter: Expr | None = None) -> Expr ... [], [dfn.functions.covar_samp(dfn.col("a"), dfn.col("b")).alias("v")]) >>> result.collect_column("v")[0].as_py() 1.0 + + >>> result = df.aggregate( + ... [], [dfn.functions.covar_samp( + ... dfn.col("a"), dfn.col("b"), + ... filter=dfn.col("a") > dfn.lit(1.0) + ... ).alias("v")]) + >>> result.collect_column("v")[0].as_py() + 0.5 """ filter_raw = filter.expr if filter is not None else None return Expr(f.covar_samp(value_y.expr, value_x.expr, filter=filter_raw)) @@ -3508,12 +3758,19 @@ def max(expression: Expr, filter: Expr | None = None) -> Expr: filter: If provided, only compute against rows for which the filter is True Examples: - --------- - >>> ctx = dfn.SessionContext() - >>> df = ctx.from_pydict({"a": [1, 2, 3]}) - >>> result = df.aggregate([], [dfn.functions.max(dfn.col("a")).alias("v")]) - >>> result.collect_column("v")[0].as_py() - 3 + >>> ctx = dfn.SessionContext() + >>> df = ctx.from_pydict({"a": [1, 2, 3]}) + >>> result = df.aggregate([], [dfn.functions.max(dfn.col("a")).alias("v")]) + >>> result.collect_column("v")[0].as_py() + 3 + + >>> result = df.aggregate( + ... [], [dfn.functions.max( + ... dfn.col("a"), + ... filter=dfn.col("a") < dfn.lit(3) + ... ).alias("v")]) + >>> result.collect_column("v")[0].as_py() + 2 """ filter_raw = filter.expr if filter is not None else None return Expr(f.max(expression.expr, filter=filter_raw)) @@ -3522,15 +3779,8 @@ def max(expression: Expr, filter: Expr | None = None) -> Expr: def mean(expression: Expr, filter: Expr | None = None) -> Expr: """Returns the average (mean) value of the argument. - This is an alias for :py:func:`avg`. - - Examples: - --------- - >>> ctx = dfn.SessionContext() - >>> df = ctx.from_pydict({"a": [1.0, 2.0, 3.0]}) - >>> result = df.aggregate([], [dfn.functions.mean(dfn.col("a")).alias("v")]) - >>> result.collect_column("v")[0].as_py() - 2.0 + See Also: + This is an alias for :py:func:`avg`. """ return avg(expression, filter) @@ -3552,12 +3802,20 @@ def median( filter: If provided, only compute against rows for which the filter is True Examples: - --------- - >>> ctx = dfn.SessionContext() - >>> df = ctx.from_pydict({"a": [1.0, 2.0, 3.0]}) - >>> result = df.aggregate([], [dfn.functions.median(dfn.col("a")).alias("v")]) - >>> result.collect_column("v")[0].as_py() - 2.0 + >>> ctx = dfn.SessionContext() + >>> df = ctx.from_pydict({"a": [1.0, 2.0, 3.0]}) + >>> result = df.aggregate([], [dfn.functions.median(dfn.col("a")).alias("v")]) + >>> result.collect_column("v")[0].as_py() + 2.0 + + >>> df = ctx.from_pydict({"a": [1.0, 1.0, 2.0, 3.0]}) + >>> result = df.aggregate( + ... [], [dfn.functions.median( + ... dfn.col("a"), distinct=True, + ... filter=dfn.col("a") > dfn.lit(0.0), + ... ).alias("v")]) + >>> result.collect_column("v")[0].as_py() + 2.0 """ filter_raw = filter.expr if filter is not None else None return Expr(f.median(expression.expr, distinct=distinct, filter=filter_raw)) @@ -3574,12 +3832,19 @@ def min(expression: Expr, filter: Expr | None = None) -> Expr: filter: If provided, only compute against rows for which the filter is True Examples: - --------- - >>> ctx = dfn.SessionContext() - >>> df = ctx.from_pydict({"a": [1, 2, 3]}) - >>> result = df.aggregate([], [dfn.functions.min(dfn.col("a")).alias("v")]) - >>> result.collect_column("v")[0].as_py() - 1 + >>> ctx = dfn.SessionContext() + >>> df = ctx.from_pydict({"a": [1, 2, 3]}) + >>> result = df.aggregate([], [dfn.functions.min(dfn.col("a")).alias("v")]) + >>> result.collect_column("v")[0].as_py() + 1 + + >>> result = df.aggregate( + ... [], [dfn.functions.min( + ... dfn.col("a"), + ... filter=dfn.col("a") > dfn.lit(1) + ... ).alias("v")]) + >>> result.collect_column("v")[0].as_py() + 2 """ filter_raw = filter.expr if filter is not None else None return Expr(f.min(expression.expr, filter=filter_raw)) @@ -3601,12 +3866,19 @@ def sum( filter: If provided, only compute against rows for which the filter is True Examples: - --------- - >>> ctx = dfn.SessionContext() - >>> df = ctx.from_pydict({"a": [1, 2, 3]}) - >>> result = df.aggregate([], [dfn.functions.sum(dfn.col("a")).alias("v")]) - >>> result.collect_column("v")[0].as_py() - 6 + >>> ctx = dfn.SessionContext() + >>> df = ctx.from_pydict({"a": [1, 2, 3]}) + >>> result = df.aggregate([], [dfn.functions.sum(dfn.col("a")).alias("v")]) + >>> result.collect_column("v")[0].as_py() + 6 + + >>> result = df.aggregate( + ... [], [dfn.functions.sum( + ... dfn.col("a"), + ... filter=dfn.col("a") > dfn.lit(1) + ... ).alias("v")]) + >>> result.collect_column("v")[0].as_py() + 5 """ filter_raw = filter.expr if filter is not None else None return Expr(f.sum(expression.expr, filter=filter_raw)) @@ -3628,6 +3900,15 @@ def stddev(expression: Expr, filter: Expr | None = None) -> Expr: >>> result = df.aggregate([], [dfn.functions.stddev(dfn.col("a")).alias("v")]) >>> result.collect_column("v")[0].as_py() 2.0 + + >>> df = ctx.from_pydict({"a": [1.0, 2.0, 4.0, 6.0]}) + >>> result = df.aggregate( + ... [], [dfn.functions.stddev( + ... dfn.col("a"), + ... filter=dfn.col("a") > dfn.lit(1.0) + ... ).alias("v")]) + >>> result.collect_column("v")[0].as_py() + 2.0 """ filter_raw = filter.expr if filter is not None else None return Expr(f.stddev(expression.expr, filter=filter_raw)) @@ -3651,6 +3932,15 @@ def stddev_pop(expression: Expr, filter: Expr | None = None) -> Expr: ... ) >>> result.collect_column("v")[0].as_py() 1.0 + + >>> df = ctx.from_pydict({"a": [0.0, 1.0, 3.0]}) + >>> result = df.aggregate( + ... [], [dfn.functions.stddev_pop( + ... dfn.col("a"), + ... filter=dfn.col("a") > dfn.lit(0.0) + ... ).alias("v")]) + >>> result.collect_column("v")[0].as_py() + 1.0 """ filter_raw = filter.expr if filter is not None else None return Expr(f.stddev_pop(expression.expr, filter=filter_raw)) @@ -3659,16 +3949,8 @@ def stddev_pop(expression: Expr, filter: Expr | None = None) -> Expr: def stddev_samp(arg: Expr, filter: Expr | None = None) -> Expr: """Computes the sample standard deviation of the argument. - This is an alias for :py:func:`stddev`. - - Examples: - >>> ctx = dfn.SessionContext() - >>> df = ctx.from_pydict({"a": [2.0, 4.0, 6.0]}) - >>> result = df.aggregate( - ... [], [dfn.functions.stddev_samp(dfn.col("a")).alias("v")] - ... ) - >>> result.collect_column("v")[0].as_py() - 2.0 + See Also: + This is an alias for :py:func:`stddev`. """ return stddev(arg, filter=filter) @@ -3676,14 +3958,8 @@ def stddev_samp(arg: Expr, filter: Expr | None = None) -> Expr: def var(expression: Expr, filter: Expr | None = None) -> Expr: """Computes the sample variance of the argument. - This is an alias for :py:func:`var_samp`. - - Examples: - >>> ctx = dfn.SessionContext() - >>> df = ctx.from_pydict({"a": [1.0, 2.0, 3.0]}) - >>> result = df.aggregate([], [dfn.functions.var(dfn.col("a")).alias("v")]) - >>> result.collect_column("v")[0].as_py() - 1.0 + See Also: + This is an alias for :py:func:`var_samp`. """ return var_samp(expression, filter) @@ -3704,6 +3980,15 @@ def var_pop(expression: Expr, filter: Expr | None = None) -> Expr: >>> result = df.aggregate([], [dfn.functions.var_pop(dfn.col("a")).alias("v")]) >>> result.collect_column("v")[0].as_py() 1.0 + + >>> df = ctx.from_pydict({"a": [-1.0, 0.0, 2.0]}) + >>> result = df.aggregate( + ... [], [dfn.functions.var_pop( + ... dfn.col("a"), + ... filter=dfn.col("a") > dfn.lit(-1.0) + ... ).alias("v")]) + >>> result.collect_column("v")[0].as_py() + 1.0 """ filter_raw = filter.expr if filter is not None else None return Expr(f.var_pop(expression.expr, filter=filter_raw)) @@ -3725,6 +4010,14 @@ def var_samp(expression: Expr, filter: Expr | None = None) -> Expr: >>> result = df.aggregate([], [dfn.functions.var_samp(dfn.col("a")).alias("v")]) >>> result.collect_column("v")[0].as_py() 1.0 + + >>> result = df.aggregate( + ... [], [dfn.functions.var_samp( + ... dfn.col("a"), + ... filter=dfn.col("a") > dfn.lit(1.0) + ... ).alias("v")]) + >>> result.collect_column("v")[0].as_py() + 0.5 """ filter_raw = filter.expr if filter is not None else None return Expr(f.var_sample(expression.expr, filter=filter_raw)) @@ -3733,16 +4026,8 @@ def var_samp(expression: Expr, filter: Expr | None = None) -> Expr: def var_sample(expression: Expr, filter: Expr | None = None) -> Expr: """Computes the sample variance of the argument. - This is an alias for :py:func:`var_samp`. - - Examples: - >>> ctx = dfn.SessionContext() - >>> df = ctx.from_pydict({"a": [1.0, 2.0, 3.0]}) - >>> result = df.aggregate( - ... [], [dfn.functions.var_sample(dfn.col("a")).alias("v")] - ... ) - >>> result.collect_column("v")[0].as_py() - 1.0 + See Also: + This is an alias for :py:func:`var_samp`. """ return var_samp(expression, filter) @@ -3772,6 +4057,14 @@ def regr_avgx( ... [], [dfn.functions.regr_avgx(dfn.col("y"), dfn.col("x")).alias("v")]) >>> result.collect_column("v")[0].as_py() 5.0 + + >>> result = df.aggregate( + ... [], [dfn.functions.regr_avgx( + ... dfn.col("y"), dfn.col("x"), + ... filter=dfn.col("y") > dfn.lit(1.0) + ... ).alias("v")]) + >>> result.collect_column("v")[0].as_py() + 5.5 """ filter_raw = filter.expr if filter is not None else None @@ -3803,6 +4096,14 @@ def regr_avgy( ... [], [dfn.functions.regr_avgy(dfn.col("y"), dfn.col("x")).alias("v")]) >>> result.collect_column("v")[0].as_py() 2.0 + + >>> result = df.aggregate( + ... [], [dfn.functions.regr_avgy( + ... dfn.col("y"), dfn.col("x"), + ... filter=dfn.col("y") > dfn.lit(1.0) + ... ).alias("v")]) + >>> result.collect_column("v")[0].as_py() + 2.5 """ filter_raw = filter.expr if filter is not None else None @@ -3834,6 +4135,14 @@ def regr_count( ... [], [dfn.functions.regr_count(dfn.col("y"), dfn.col("x")).alias("v")]) >>> result.collect_column("v")[0].as_py() 3 + + >>> result = df.aggregate( + ... [], [dfn.functions.regr_count( + ... dfn.col("y"), dfn.col("x"), + ... filter=dfn.col("y") > dfn.lit(1.0) + ... ).alias("v")]) + >>> result.collect_column("v")[0].as_py() + 2 """ filter_raw = filter.expr if filter is not None else None @@ -3866,6 +4175,15 @@ def regr_intercept( ... [dfn.functions.regr_intercept(dfn.col("y"), dfn.col("x")).alias("v")]) >>> result.collect_column("v")[0].as_py() 0.0 + + >>> result = df.aggregate( + ... [], + ... [dfn.functions.regr_intercept( + ... dfn.col("y"), dfn.col("x"), + ... filter=dfn.col("y") > dfn.lit(2.0) + ... ).alias("v")]) + >>> result.collect_column("v")[0].as_py() + 0.0 """ filter_raw = filter.expr if filter is not None else None @@ -3897,6 +4215,14 @@ def regr_r2( ... [], [dfn.functions.regr_r2(dfn.col("y"), dfn.col("x")).alias("v")]) >>> result.collect_column("v")[0].as_py() 1.0 + + >>> result = df.aggregate( + ... [], [dfn.functions.regr_r2( + ... dfn.col("y"), dfn.col("x"), + ... filter=dfn.col("y") > dfn.lit(2.0) + ... ).alias("v")]) + >>> result.collect_column("v")[0].as_py() + 1.0 """ filter_raw = filter.expr if filter is not None else None @@ -3928,6 +4254,14 @@ def regr_slope( ... [], [dfn.functions.regr_slope(dfn.col("y"), dfn.col("x")).alias("v")]) >>> result.collect_column("v")[0].as_py() 2.0 + + >>> result = df.aggregate( + ... [], [dfn.functions.regr_slope( + ... dfn.col("y"), dfn.col("x"), + ... filter=dfn.col("y") > dfn.lit(2.0) + ... ).alias("v")]) + >>> result.collect_column("v")[0].as_py() + 2.0 """ filter_raw = filter.expr if filter is not None else None @@ -3959,6 +4293,14 @@ def regr_sxx( ... [], [dfn.functions.regr_sxx(dfn.col("y"), dfn.col("x")).alias("v")]) >>> result.collect_column("v")[0].as_py() 2.0 + + >>> result = df.aggregate( + ... [], [dfn.functions.regr_sxx( + ... dfn.col("y"), dfn.col("x"), + ... filter=dfn.col("y") > dfn.lit(1.0) + ... ).alias("v")]) + >>> result.collect_column("v")[0].as_py() + 0.5 """ filter_raw = filter.expr if filter is not None else None @@ -3990,6 +4332,14 @@ def regr_sxy( ... [], [dfn.functions.regr_sxy(dfn.col("y"), dfn.col("x")).alias("v")]) >>> result.collect_column("v")[0].as_py() 2.0 + + >>> result = df.aggregate( + ... [], [dfn.functions.regr_sxy( + ... dfn.col("y"), dfn.col("x"), + ... filter=dfn.col("y") > dfn.lit(1.0) + ... ).alias("v")]) + >>> result.collect_column("v")[0].as_py() + 0.5 """ filter_raw = filter.expr if filter is not None else None @@ -4021,6 +4371,14 @@ def regr_syy( ... [], [dfn.functions.regr_syy(dfn.col("y"), dfn.col("x")).alias("v")]) >>> result.collect_column("v")[0].as_py() 2.0 + + >>> result = df.aggregate( + ... [], [dfn.functions.regr_syy( + ... dfn.col("y"), dfn.col("x"), + ... filter=dfn.col("y") > dfn.lit(1.0) + ... ).alias("v")]) + >>> result.collect_column("v")[0].as_py() + 0.5 """ filter_raw = filter.expr if filter is not None else None @@ -4047,9 +4405,25 @@ def first_value( column names or expressions. null_treatment: Assign whether to respect or ignore null values. - For example:: + Examples: + >>> ctx = dfn.SessionContext() + >>> df = ctx.from_pydict({"a": [10, 20, 30]}) + >>> result = df.aggregate( + ... [], [dfn.functions.first_value(dfn.col("a")).alias("v")] + ... ) + >>> result.collect_column("v")[0].as_py() + 10 - df.aggregate([], first_value(col("a"), order_by="ts")) + >>> df = ctx.from_pydict({"a": [None, 20, 10]}) + >>> result = df.aggregate( + ... [], [dfn.functions.first_value( + ... dfn.col("a"), + ... filter=dfn.col("a") > dfn.lit(10), + ... order_by="a", + ... null_treatment=dfn.common.NullTreatment.IGNORE_NULLS, + ... ).alias("v")]) + >>> result.collect_column("v")[0].as_py() + 20 """ order_by_raw = sort_list_to_raw_sort_list(order_by) filter_raw = filter.expr if filter is not None else None @@ -4084,9 +4458,25 @@ def last_value( column names or expressions. null_treatment: Assign whether to respect or ignore null values. - For example:: + Examples: + >>> ctx = dfn.SessionContext() + >>> df = ctx.from_pydict({"a": [10, 20, 30]}) + >>> result = df.aggregate( + ... [], [dfn.functions.last_value(dfn.col("a")).alias("v")] + ... ) + >>> result.collect_column("v")[0].as_py() + 30 - df.aggregate([], last_value(col("a"), order_by="ts")) + >>> df = ctx.from_pydict({"a": [None, 20, 10]}) + >>> result = df.aggregate( + ... [], [dfn.functions.last_value( + ... dfn.col("a"), + ... filter=dfn.col("a") > dfn.lit(10), + ... order_by="a", + ... null_treatment=dfn.common.NullTreatment.IGNORE_NULLS, + ... ).alias("v")]) + >>> result.collect_column("v")[0].as_py() + 20 """ order_by_raw = sort_list_to_raw_sort_list(order_by) filter_raw = filter.expr if filter is not None else None @@ -4123,9 +4513,25 @@ def nth_value( column names or expressions. null_treatment: Assign whether to respect or ignore null values. - For example:: + Examples: + >>> ctx = dfn.SessionContext() + >>> df = ctx.from_pydict({"a": [10, 20, 30]}) + >>> result = df.aggregate( + ... [], [dfn.functions.nth_value(dfn.col("a"), 2).alias("v")] + ... ) + >>> result.collect_column("v")[0].as_py() + 20 - df.aggregate([], nth_value(col("a"), 2, order_by="ts")) + >>> df = ctx.from_pydict({"a": [None, 20, 10]}) + >>> result = df.aggregate( + ... [], [dfn.functions.nth_value( + ... dfn.col("a"), 1, + ... filter=dfn.col("a") > dfn.lit(10), + ... order_by="a", + ... null_treatment=dfn.common.NullTreatment.IGNORE_NULLS, + ... ).alias("v")]) + >>> result.collect_column("v")[0].as_py() + 20 """ order_by_raw = sort_list_to_raw_sort_list(order_by) filter_raw = filter.expr if filter is not None else None @@ -4154,12 +4560,20 @@ def bit_and(expression: Expr, filter: Expr | None = None) -> Expr: filter: If provided, only compute against rows for which the filter is True Examples: - --------- - >>> ctx = dfn.SessionContext() - >>> df = ctx.from_pydict({"a": [7, 3]}) - >>> result = df.aggregate([], [dfn.functions.bit_and(dfn.col("a")).alias("v")]) - >>> result.collect_column("v")[0].as_py() - 3 + >>> ctx = dfn.SessionContext() + >>> df = ctx.from_pydict({"a": [7, 3]}) + >>> result = df.aggregate([], [dfn.functions.bit_and(dfn.col("a")).alias("v")]) + >>> result.collect_column("v")[0].as_py() + 3 + + >>> df = ctx.from_pydict({"a": [7, 5, 3]}) + >>> result = df.aggregate( + ... [], [dfn.functions.bit_and( + ... dfn.col("a"), + ... filter=dfn.col("a") > dfn.lit(3) + ... ).alias("v")]) + >>> result.collect_column("v")[0].as_py() + 5 """ filter_raw = filter.expr if filter is not None else None return Expr(f.bit_and(expression.expr, filter=filter_raw)) @@ -4178,12 +4592,20 @@ def bit_or(expression: Expr, filter: Expr | None = None) -> Expr: filter: If provided, only compute against rows for which the filter is True Examples: - --------- - >>> ctx = dfn.SessionContext() - >>> df = ctx.from_pydict({"a": [1, 2]}) - >>> result = df.aggregate([], [dfn.functions.bit_or(dfn.col("a")).alias("v")]) - >>> result.collect_column("v")[0].as_py() - 3 + >>> ctx = dfn.SessionContext() + >>> df = ctx.from_pydict({"a": [1, 2]}) + >>> result = df.aggregate([], [dfn.functions.bit_or(dfn.col("a")).alias("v")]) + >>> result.collect_column("v")[0].as_py() + 3 + + >>> df = ctx.from_pydict({"a": [1, 2, 4]}) + >>> result = df.aggregate( + ... [], [dfn.functions.bit_or( + ... dfn.col("a"), + ... filter=dfn.col("a") > dfn.lit(1) + ... ).alias("v")]) + >>> result.collect_column("v")[0].as_py() + 6 """ filter_raw = filter.expr if filter is not None else None return Expr(f.bit_or(expression.expr, filter=filter_raw)) @@ -4205,12 +4627,20 @@ def bit_xor( filter: If provided, only compute against rows for which the filter is True Examples: - --------- - >>> ctx = dfn.SessionContext() - >>> df = ctx.from_pydict({"a": [5, 3]}) - >>> result = df.aggregate([], [dfn.functions.bit_xor(dfn.col("a")).alias("v")]) - >>> result.collect_column("v")[0].as_py() - 6 + >>> ctx = dfn.SessionContext() + >>> df = ctx.from_pydict({"a": [5, 3]}) + >>> result = df.aggregate([], [dfn.functions.bit_xor(dfn.col("a")).alias("v")]) + >>> result.collect_column("v")[0].as_py() + 6 + + >>> df = ctx.from_pydict({"a": [5, 5, 3]}) + >>> result = df.aggregate( + ... [], [dfn.functions.bit_xor( + ... dfn.col("a"), distinct=True, + ... filter=dfn.col("a") > dfn.lit(0), + ... ).alias("v")]) + >>> result.collect_column("v")[0].as_py() + 6 """ filter_raw = filter.expr if filter is not None else None return Expr(f.bit_xor(expression.expr, distinct=distinct, filter=filter_raw)) @@ -4230,12 +4660,21 @@ def bool_and(expression: Expr, filter: Expr | None = None) -> Expr: filter: If provided, only compute against rows for which the filter is True Examples: - --------- - >>> ctx = dfn.SessionContext() - >>> df = ctx.from_pydict({"a": [True, True, False]}) - >>> result = df.aggregate([], [dfn.functions.bool_and(dfn.col("a")).alias("v")]) - >>> result.collect_column("v")[0].as_py() - False + >>> ctx = dfn.SessionContext() + >>> df = ctx.from_pydict({"a": [True, True, False]}) + >>> result = df.aggregate([], [dfn.functions.bool_and(dfn.col("a")).alias("v")]) + >>> result.collect_column("v")[0].as_py() + False + + >>> df = ctx.from_pydict( + ... {"a": [True, True, False], "b": [1, 2, 3]}) + >>> result = df.aggregate( + ... [], [dfn.functions.bool_and( + ... dfn.col("a"), + ... filter=dfn.col("b") < dfn.lit(3) + ... ).alias("v")]) + >>> result.collect_column("v")[0].as_py() + True """ filter_raw = filter.expr if filter is not None else None return Expr(f.bool_and(expression.expr, filter=filter_raw)) @@ -4255,12 +4694,21 @@ def bool_or(expression: Expr, filter: Expr | None = None) -> Expr: filter: If provided, only compute against rows for which the filter is True Examples: - --------- - >>> ctx = dfn.SessionContext() - >>> df = ctx.from_pydict({"a": [False, False, True]}) - >>> result = df.aggregate([], [dfn.functions.bool_or(dfn.col("a")).alias("v")]) - >>> result.collect_column("v")[0].as_py() - True + >>> ctx = dfn.SessionContext() + >>> df = ctx.from_pydict({"a": [False, False, True]}) + >>> result = df.aggregate([], [dfn.functions.bool_or(dfn.col("a")).alias("v")]) + >>> result.collect_column("v")[0].as_py() + True + + >>> df = ctx.from_pydict( + ... {"a": [False, False, True], "b": [1, 2, 3]}) + >>> result = df.aggregate( + ... [], [dfn.functions.bool_or( + ... dfn.col("a"), + ... filter=dfn.col("b") < dfn.lit(3) + ... ).alias("v")]) + >>> result.collect_column("v")[0].as_py() + False """ filter_raw = filter.expr if filter is not None else None return Expr(f.bool_or(expression.expr, filter=filter_raw)) @@ -4303,9 +4751,24 @@ def lead( order_by: Set ordering within the window frame. Accepts column names or expressions. - For example:: + Examples: + >>> ctx = dfn.SessionContext() + >>> df = ctx.from_pydict({"a": [1, 2, 3]}) + >>> result = df.select( + ... dfn.col("a"), dfn.functions.lead(dfn.col("a"), shift_offset=1, + ... default_value=0, order_by="a").alias("lead")) + >>> result.sort(dfn.col("a")).collect_column("lead").to_pylist() + [2, 3, 0] - lead(col("b"), order_by="ts") + >>> df = ctx.from_pydict({"g": ["a", "a", "b"], "v": [1, 2, 3]}) + >>> result = df.select( + ... dfn.col("g"), dfn.col("v"), + ... dfn.functions.lead( + ... dfn.col("v"), shift_offset=1, default_value=0, + ... partition_by=dfn.col("g"), order_by="v", + ... ).alias("lead")) + >>> result.sort(dfn.col("g"), dfn.col("v")).collect_column("lead").to_pylist() + [2, 0, 0] """ if not isinstance(default_value, pa.Scalar) and default_value is not None: default_value = pa.scalar(default_value) @@ -4358,9 +4821,24 @@ def lag( order_by: Set ordering within the window frame. Accepts column names or expressions. - For example:: + Examples: + >>> ctx = dfn.SessionContext() + >>> df = ctx.from_pydict({"a": [1, 2, 3]}) + >>> result = df.select( + ... dfn.col("a"), dfn.functions.lag(dfn.col("a"), shift_offset=1, + ... default_value=0, order_by="a").alias("lag")) + >>> result.sort(dfn.col("a")).collect_column("lag").to_pylist() + [0, 1, 2] - lag(col("b"), order_by="ts") + >>> df = ctx.from_pydict({"g": ["a", "a", "b"], "v": [1, 2, 3]}) + >>> result = df.select( + ... dfn.col("g"), dfn.col("v"), + ... dfn.functions.lag( + ... dfn.col("v"), shift_offset=1, default_value=0, + ... partition_by=dfn.col("g"), order_by="v", + ... ).alias("lag")) + >>> result.sort(dfn.col("g"), dfn.col("v")).collect_column("lag").to_pylist() + [0, 1, 0] """ if not isinstance(default_value, pa.Scalar): default_value = pa.scalar(default_value) @@ -4403,9 +4881,23 @@ def row_number( order_by: Set ordering within the window frame. Accepts column names or expressions. - For example:: + Examples: + >>> ctx = dfn.SessionContext() + >>> df = ctx.from_pydict({"a": [10, 20, 30]}) + >>> result = df.select( + ... dfn.col("a"), dfn.functions.row_number(order_by="a").alias("rn")) + >>> result.sort(dfn.col("a")).collect_column("rn").to_pylist() + [1, 2, 3] - row_number(order_by="points") + >>> df = ctx.from_pydict( + ... {"g": ["a", "a", "b", "b"], "v": [1, 2, 3, 4]}) + >>> result = df.select( + ... dfn.col("g"), dfn.col("v"), + ... dfn.functions.row_number( + ... partition_by=dfn.col("g"), order_by="v", + ... ).alias("rn")) + >>> result.sort(dfn.col("g"), dfn.col("v")).collect_column("rn").to_pylist() + [1, 2, 1, 2] """ partition_by_raw = expr_list_to_raw_expr_list(partition_by) order_by_raw = sort_list_to_raw_sort_list(order_by) @@ -4447,9 +4939,24 @@ def rank( order_by: Set ordering within the window frame. Accepts column names or expressions. - For example:: + Examples: + >>> ctx = dfn.SessionContext() + >>> df = ctx.from_pydict({"a": [10, 10, 20]}) + >>> result = df.select( + ... dfn.col("a"), dfn.functions.rank(order_by="a").alias("rnk") + ... ) + >>> result.sort(dfn.col("a")).collect_column("rnk").to_pylist() + [1, 1, 3] - rank(order_by="points") + >>> df = ctx.from_pydict( + ... {"g": ["a", "a", "b", "b"], "v": [1, 1, 2, 3]}) + >>> result = df.select( + ... dfn.col("g"), dfn.col("v"), + ... dfn.functions.rank( + ... partition_by=dfn.col("g"), order_by="v", + ... ).alias("rnk")) + >>> result.sort(dfn.col("g"), dfn.col("v")).collect_column("rnk").to_pylist() + [1, 1, 1, 2] """ partition_by_raw = expr_list_to_raw_expr_list(partition_by) order_by_raw = sort_list_to_raw_sort_list(order_by) @@ -4486,9 +4993,23 @@ def dense_rank( order_by: Set ordering within the window frame. Accepts column names or expressions. - For example:: + Examples: + >>> ctx = dfn.SessionContext() + >>> df = ctx.from_pydict({"a": [10, 10, 20]}) + >>> result = df.select( + ... dfn.col("a"), dfn.functions.dense_rank(order_by="a").alias("dr")) + >>> result.sort(dfn.col("a")).collect_column("dr").to_pylist() + [1, 1, 2] - dense_rank(order_by="points") + >>> df = ctx.from_pydict( + ... {"g": ["a", "a", "b", "b"], "v": [1, 1, 2, 3]}) + >>> result = df.select( + ... dfn.col("g"), dfn.col("v"), + ... dfn.functions.dense_rank( + ... partition_by=dfn.col("g"), order_by="v", + ... ).alias("dr")) + >>> result.sort(dfn.col("g"), dfn.col("v")).collect_column("dr").to_pylist() + [1, 1, 1, 2] """ partition_by_raw = expr_list_to_raw_expr_list(partition_by) order_by_raw = sort_list_to_raw_sort_list(order_by) @@ -4526,9 +5047,24 @@ def percent_rank( order_by: Set ordering within the window frame. Accepts column names or expressions. - For example:: - percent_rank(order_by="points") + Examples: + >>> ctx = dfn.SessionContext() + >>> df = ctx.from_pydict({"a": [10, 20, 30]}) + >>> result = df.select( + ... dfn.col("a"), dfn.functions.percent_rank(order_by="a").alias("pr")) + >>> result.sort(dfn.col("a")).collect_column("pr").to_pylist() + [0.0, 0.5, 1.0] + + >>> df = ctx.from_pydict( + ... {"g": ["a", "a", "a", "b", "b"], "v": [1, 2, 3, 4, 5]}) + >>> result = df.select( + ... dfn.col("g"), dfn.col("v"), + ... dfn.functions.percent_rank( + ... partition_by=dfn.col("g"), order_by="v", + ... ).alias("pr")) + >>> result.sort(dfn.col("g"), dfn.col("v")).collect_column("pr").to_pylist() + [0.0, 0.5, 1.0, 0.0, 1.0] """ partition_by_raw = expr_list_to_raw_expr_list(partition_by) order_by_raw = sort_list_to_raw_sort_list(order_by) @@ -4566,9 +5102,27 @@ def cume_dist( order_by: Set ordering within the window frame. Accepts column names or expressions. - For example:: + Examples: + >>> ctx = dfn.SessionContext() + >>> df = ctx.from_pydict({"a": [1., 2., 2., 3.]}) + >>> result = df.select( + ... dfn.col("a"), + ... dfn.functions.cume_dist( + ... order_by="a" + ... ).alias("cd") + ... ) + >>> result.collect_column("cd").to_pylist() + [0.25..., 0.75..., 0.75..., 1.0...] - cume_dist(order_by="points") + >>> df = ctx.from_pydict( + ... {"g": ["a", "a", "b", "b"], "v": [1, 2, 3, 4]}) + >>> result = df.select( + ... dfn.col("g"), dfn.col("v"), + ... dfn.functions.cume_dist( + ... partition_by=dfn.col("g"), order_by="v", + ... ).alias("cd")) + >>> result.sort(dfn.col("g"), dfn.col("v")).collect_column("cd").to_pylist() + [0.5, 1.0, 0.5, 1.0] """ partition_by_raw = expr_list_to_raw_expr_list(partition_by) order_by_raw = sort_list_to_raw_sort_list(order_by) @@ -4610,9 +5164,23 @@ def ntile( order_by: Set ordering within the window frame. Accepts column names or expressions. - For example:: + Examples: + >>> ctx = dfn.SessionContext() + >>> df = ctx.from_pydict({"a": [10, 20, 30, 40]}) + >>> result = df.select( + ... dfn.col("a"), dfn.functions.ntile(2, order_by="a").alias("nt")) + >>> result.sort(dfn.col("a")).collect_column("nt").to_pylist() + [1, 1, 2, 2] - ntile(3, order_by="points") + >>> df = ctx.from_pydict( + ... {"g": ["a", "a", "b", "b"], "v": [1, 2, 3, 4]}) + >>> result = df.select( + ... dfn.col("g"), dfn.col("v"), + ... dfn.functions.ntile( + ... 2, partition_by=dfn.col("g"), order_by="v", + ... ).alias("nt")) + >>> result.sort(dfn.col("g"), dfn.col("v")).collect_column("nt").to_pylist() + [1, 2, 1, 2] """ partition_by_raw = expr_list_to_raw_expr_list(partition_by) order_by_raw = sort_list_to_raw_sort_list(order_by) @@ -4648,18 +5216,24 @@ def string_agg( order_by: Set the ordering of the expression to evaluate. Accepts column names or expressions. - For example:: - - df.aggregate([], string_agg(col("a"), ",", order_by="b")) - Examples: - --------- - >>> ctx = dfn.SessionContext() - >>> df = ctx.from_pydict({"a": ["x", "y", "z"]}) - >>> result = df.aggregate( - ... [], [dfn.functions.string_agg(dfn.col("a"), ",", order_by="a").alias("s")]) - >>> result.collect_column("s")[0].as_py() - 'x,y,z' + >>> ctx = dfn.SessionContext() + >>> df = ctx.from_pydict({"a": ["x", "y", "z"]}) + >>> result = df.aggregate( + ... [], [dfn.functions.string_agg( + ... dfn.col("a"), ",", order_by="a" + ... ).alias("s")]) + >>> result.collect_column("s")[0].as_py() + 'x,y,z' + + >>> result = df.aggregate( + ... [], [dfn.functions.string_agg( + ... dfn.col("a"), ",", + ... filter=dfn.col("a") > dfn.lit("x"), + ... order_by="a", + ... ).alias("s")]) + >>> result.collect_column("s")[0].as_py() + 'y,z' """ order_by_raw = sort_list_to_raw_sort_list(order_by) filter_raw = filter.expr if filter is not None else None