Escaping Control Characters¶

In [ ]:

from pyspark.sql import SparkSession

spark = SparkSession.builder.getOrCreate()

%load_ext jupyterlab_sql_editor.ipython_magic.sparksql

from pyspark.sql import SparkSession spark = SparkSession.builder.getOrCreate() %load_ext jupyterlab_sql_editor.ipython_magic.sparksql

In [2]:

spark.sql('''
--start-sparksql
SELECT
    '\r\t\b\n\t\n' AS a,
    1 AS num,
    ARRAY('\t123', 'ab\nc') AS str_array,
    '\t' AS tab,
    '\\t' AS backslash_and_t,
    '\\\t' AS backslash_and_tab,
    '\\\\t' AS two_backslash_and_t,
    '\\\\\t' AS two_backslash_and_tab
--end-sparksql
''').show()

spark.sql(''' --start-sparksql SELECT '\r\t\b\n\t\n' AS a, 1 AS num, ARRAY('\t123', 'ab\nc') AS str_array, '\t' AS tab, '\\t' AS backslash_and_t, '\\\t' AS backslash_and_tab, '\\\\t' AS two_backslash_and_t, '\\\\\t' AS two_backslash_and_tab --end-sparksql ''').show()

+------+---+------------+---+---------------+-----------------+-------------------+---------------------+
|     a|num|   str_array|tab|backslash_and_t|backslash_and_tab|two_backslash_and_t|two_backslash_and_tab|
+------+---+------------+---+---------------+-----------------+-------------------+---------------------+
|
	
|  1|[	123, ab
c]|  	|              	|                	|                 \t|                   \	|
+------+---+------------+---+---------------+-----------------+-------------------+---------------------+

In [3]:

%%sparksql --output html --show-nonprinting
SELECT
    '\r\t\b\n\t\n' AS a,
    1 AS num,
    ARRAY('\t123', 'ab\nc') AS str_array,
    '\t' AS tab,
    '\\t' AS backslash_and_t,
    '\\\t' AS backslash_and_tab,
    '\\\\t' AS two_backslash_and_t,
    '\\\\\t' AS two_backslash_and_tab

%%sparksql --output html --show-nonprinting SELECT '\r\t\b\n\t\n' AS a, 1 AS num, ARRAY('\t123', 'ab\nc') AS str_array, '\t' AS tab, '\\t' AS backslash_and_t, '\\\t' AS backslash_and_tab, '\\\\t' AS two_backslash_and_t, '\\\\\t' AS two_backslash_and_tab

TTL -1 seconds expired, re-generating schema file: /tmp/sparkdb.schema.json
Generating schema file: /tmp/sparkdb.schema.json
Schema file updated: /tmp/sparkdb.schema.json

Out[3]:

a	num	str_array	tab	backslash_and_t	backslash_and_tab	two_backslash_and_t	two_backslash_and_tab
\x0d\x09\x08\x0a\x09\x0a	1	['\t123', 'ab\nc']	\x09	\t	\\x09	\\t	\\\x09

In [4]:

%%sparksql --output grid --show-nonprinting
SELECT
    '\r\t\b\n\t\n' AS a,
    1 AS num,
    ARRAY('\t123', 'ab\nc') AS str_array,
    '\t' AS tab,
    '\\t' AS backslash_and_t,
    '\\\t' AS backslash_and_tab,
    '\\\\t' AS two_backslash_and_t,
    '\\\\\t' AS two_backslash_and_tab

%%sparksql --output grid --show-nonprinting SELECT '\r\t\b\n\t\n' AS a, 1 AS num, ARRAY('\t123', 'ab\nc') AS str_array, '\t' AS tab, '\\t' AS backslash_and_t, '\\\t' AS backslash_and_tab, '\\\\t' AS two_backslash_and_t, '\\\\\t' AS two_backslash_and_tab

TTL -1 seconds expired, re-generating schema file: /tmp/sparkdb.schema.json
Generating schema file: /tmp/sparkdb.schema.json
Schema file updated: /tmp/sparkdb.schema.json

DataGrid(auto_fit_params={'area': 'all', 'padding': 30, 'numCols': None}, corner_renderer=None, default_render…

In [5]:

%%sparksql --output json --show-nonprinting
SELECT
    '\r\t\b\n\t\n' AS a,
    1 AS num,
    ARRAY('\t123', 'ab\nc') AS str_array,
    '\t' AS tab,
    '\\t' AS backslash_and_t,
    '\\\t' AS backslash_and_tab,
    '\\\\t' AS two_backslash_and_t,
    '\\\\\t' AS two_backslash_and_tab

%%sparksql --output json --show-nonprinting SELECT '\r\t\b\n\t\n' AS a, 1 AS num, ARRAY('\t123', 'ab\nc') AS str_array, '\t' AS tab, '\\t' AS backslash_and_t, '\\\t' AS backslash_and_tab, '\\\\t' AS two_backslash_and_t, '\\\\\t' AS two_backslash_and_tab

TTL -1 seconds expired, re-generating schema file: /tmp/sparkdb.schema.json
Generating schema file: /tmp/sparkdb.schema.json
Schema file updated: /tmp/sparkdb.schema.json

Out[5]:

<IPython.core.display.JSON object>