Spark Syntax Demo¶

In [ ]:

%%sparksql --dataframe df -c --eager -v MY_VIEW --limit 12 -f adir/out.json -t 60
-- cell magic
SELECT *
FROM student AS cellmagic

%%sparksql --dataframe df -c --eager -v MY_VIEW --limit 12 -f adir/out.json -t 60 -- cell magic SELECT * FROM student AS cellmagic

In [ ]:

%sparksql -d df SELECT * from student where x=1

%sparksql -d df SELECT * from student where x=1

In [ ]:

%sparksql --dataframe df -c --eager -v MY_VIEW --limit 12 -f adir/out.json -t 60 SELECT * from student -- line magic using no argument options like --eager

%sparksql --dataframe df -c --eager -v MY_VIEW --limit 12 -f adir/out.json -t 60 SELECT * from student -- line magic using no argument options like --eager

In [ ]:

%sparksql SELECT * FROM tab WHERE x = 1


# select is not be highlighted anymore, but it does require a blank line (line 3 above)
%sparksql  --cache   SELECT * FROM tab WHERE x = 2

%sparksql --cache SELECT * FROM tab WHERE x = 3

%sparksql SELECT * FROM tab WHERE x = 1 # select is not be highlighted anymore, but it does require a blank line (line 3 above) %sparksql --cache SELECT * FROM tab WHERE x = 2 %sparksql --cache SELECT * FROM tab WHERE x = 3

In [ ]:

# mix python and SQL

# python import
import pyspark

%sparksql SELECT s.age FROM student AS linemagic1 -- line magic in a mix cell

%sparksql SELECT s.age FROM student AS linemagic2 -- another line magic in a mix cell

# a python string with SQL within it
sql = '''
--start-sparksql anything here is ignored and is not included in the SQL statement for LSP
select s.age from student as frompythonstring
--end-sparksql
'''

# back to python
print(sql)

spark.sql(sql).show()

# mix python and SQL # python import import pyspark %sparksql SELECT s.age FROM student AS linemagic1 -- line magic in a mix cell %sparksql SELECT s.age FROM student AS linemagic2 -- another line magic in a mix cell # a python string with SQL within it sql = ''' --start-sparksql anything here is ignored and is not included in the SQL statement for LSP select s.age from student as frompythonstring --end-sparksql ''' # back to python print(sql) spark.sql(sql).show()