pysolr - connect to solr via vpn
solr = pysolr.Solr('http://***', timeout=10, auth=('username','password'))
cursor = '*'
df1 = pd.DataFrame()
# Batch computation
# Fetch 100 rows per 10 times: 10 * 100 --> using nextCursorMark
for x in range(0,10):
results = solr.search('var_1:* AND var_2":true AND var_year:2020', **{
'fl': 'id,var_3,var_4',
'cursorMark': cursor,
'sort': 'id desc',
'rows': '100',
'wt':'csv'
})
cursor = results.nextCursorMark
df1 = df1.append(results.docs, ignore_index=True)