{ "cells": [ { "cell_type": "markdown", "metadata": {}, "source": [ "# Initial Setup" ] }, { "cell_type": "code", "execution_count": 1, "metadata": {}, "outputs": [], "source": [ "import os\n", "import numpy as np\n", "import pandas as pd\n", "from dbfread import DBF\n", "from sqlalchemy import create_engine\n", "from tqdm import tqdm" ] }, { "cell_type": "code", "execution_count": 2, "metadata": {}, "outputs": [], "source": [ "server = '43.224.110.84' \n", "database = 'tempBR' \n", "username = 'sa' \n", "password = '8qrVA9sb'\n", "\n", "engine = create_engine(\n", " f'mssql+pyodbc://{username}:{password}@{server}/{database}?driver=ODBC+Driver+17+for+SQL+Server'\n", ")" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Check these before proceeding" ] }, { "cell_type": "code", "execution_count": 3, "metadata": {}, "outputs": [], "source": [ "exam_id = 3 # take from db\n", "exam_type = 'HSC'\n", "\n", "slots_path = 'data/H_type/Raw_scan'\n", "records_path = 'data/H_type/Raw_scan'" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Database write setup" ] }, { "cell_type": "code", "execution_count": 4, "metadata": {}, "outputs": [], "source": [ "chunksize = 1000\n", "method = 'multi'" ] }, { "cell_type": "code", "execution_count": 13, "metadata": {}, "outputs": [], "source": [ "examiners_df = pd.read_sql(\n", " \"\\\n", " SELECT examiner_code, examiner_id\\\n", " FROM Examiners\\\n", " \",\n", " engine,\n", " index_col='examiner_code'\n", ")" ] }, { "cell_type": "code", "execution_count": 6, "metadata": {}, "outputs": [], "source": [ "cqmid_bundle_id_start = pd.read_sql(\n", " \"\\\n", " SELECT max(cqmid_bundle_id) + 1\\\n", " FROM CQMidBundles\\\n", " \",\n", " engine\n", ").iloc[0,0]\n", "\n", "cqmid_slot_id_start = pd.read_sql(\n", " \"\\\n", " SELECT max(cqmid_slot_id) + 1\\\n", " FROM CQMidSlots\\\n", " \",\n", " engine\n", ").iloc[0,0]\n", "\n", "cqmid_record_id_start = pd.read_sql(\n", " \"\\\n", " SELECT max(cqmid_record_id) + 1\\\n", " FROM CQMidRecords\\\n", " \",\n", " engine\n", ").iloc[0,0]" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Testing Codes" ] }, { "cell_type": "code", "execution_count": 15, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
subject_codemachine_nobundle_nolocal_solver_idglobal_solver_idstatusexam_idno_of_erroneous_scriptlocal_start_datetimelocal_end_datetimesolving_order
cqmid_bundle_id
1421109A010300
\n", "
" ], "text/plain": [ " subject_code machine_no bundle_no local_solver_id \\\n", "cqmid_bundle_id \n", "1421 109 A 01 \n", "\n", " global_solver_id status exam_id no_of_erroneous_script \\\n", "cqmid_bundle_id \n", "1421 0 3 0 \n", "\n", " local_start_datetime local_end_datetime solving_order \n", "cqmid_bundle_id \n", "1421 0 " ] }, "execution_count": 15, "metadata": {}, "output_type": "execute_result" } ], "source": [ "bundle_code = 'D109HA01'\n", "bundle_df = pd.DataFrame({}, index=[0])\n", "bundle_df['subject_code'] = bundle_code[1:4]\n", "# bundle_df['script_type'] = bundle_code[4:5]\n", "bundle_df['machine_no'] = bundle_code[5:6]\n", "bundle_df['bundle_no'] = bundle_code[6:8]\n", "bundle_df['local_solver_id'] = ''\n", "bundle_df['global_solver_id'] = ''\n", "bundle_df['status'] = 0\n", "bundle_df['exam_id'] = exam_id\n", "bundle_df['no_of_erroneous_script'] = 0\n", "bundle_df['local_start_datetime'] = ''\n", "bundle_df['local_end_datetime'] = ''\n", "bundle_df['solving_order'] = 0\n", "bundle_df.index += cqmid_slot_id_start\n", "bundle_df.index.name = 'cqmid_bundle_id'\n", "bundle_df" ] }, { "cell_type": "code", "execution_count": 23, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
examiner_idslot_nosubject_codecqmid_bundle_idtotal_scriptend_slactual_tsno_of_rejectedstatus
cqmid_slot_id
1421200109142120002002000
14211130109142120002002000
14212470109142120002002000
14221874109142120002002000
14223194109142120002002000
14231907109142120002002000
14233227109142120002002000
14242205109142125002502500
14243485109142125002502500
14252182109142125001002500
14253452109142125001002500
14262309109142125001752500
14263599109142125001752500
14272193109142125302532530
14273463109142125302532530
\n", "
" ], "text/plain": [ " examiner_id slot_no subject_code cqmid_bundle_id \\\n", "cqmid_slot_id \n", "1421 20 0 109 1421 \n", "1421 113 0 109 1421 \n", "1421 247 0 109 1421 \n", "1422 187 4 109 1421 \n", "1422 319 4 109 1421 \n", "1423 190 7 109 1421 \n", "1423 322 7 109 1421 \n", "1424 220 5 109 1421 \n", "1424 348 5 109 1421 \n", "1425 218 2 109 1421 \n", "1425 345 2 109 1421 \n", "1426 230 9 109 1421 \n", "1426 359 9 109 1421 \n", "1427 219 3 109 1421 \n", "1427 346 3 109 1421 \n", "\n", " total_script end_sl actual_ts no_of_rejected status \n", "cqmid_slot_id \n", "1421 200 0200 200 0 \n", "1421 200 0200 200 0 \n", "1421 200 0200 200 0 \n", "1422 200 0200 200 0 \n", "1422 200 0200 200 0 \n", "1423 200 0200 200 0 \n", "1423 200 0200 200 0 \n", "1424 250 0250 250 0 \n", "1424 250 0250 250 0 \n", "1425 250 0100 250 0 \n", "1425 250 0100 250 0 \n", "1426 250 0175 250 0 \n", "1426 250 0175 250 0 \n", "1427 253 0253 253 0 \n", "1427 253 0253 253 0 " ] }, "execution_count": 23, "metadata": {}, "output_type": "execute_result" } ], "source": [ "slots_file = f'{bundle_code}.PKT'\n", "df = pd.DataFrame(DBF(f'{slots_path}/{slots_file}'))\n", "slots_df = pd.DataFrame()\n", "slots_df['examiner_code'] = df['PKT_NO'].str[0:4]\n", "slots_df = slots_df.join(examiners_df, on='examiner_code')[['examiner_id']]\n", "slots_df['slot_no'] = df['PKT_NO'].str[3:5]\n", "slots_df['subject_code'] = bundle_df['subject_code'].iloc[0]\n", "slots_df['cqmid_bundle_id'] = bundle_df.index[0]\n", "slots_df['total_script'] = df['TS_TOTAL']\n", "slots_df['end_sl'] = df['END_SL']\n", "slots_df['actual_ts'] = df['SCANNED']\n", "slots_df['no_of_rejected'] = df['MARK_REJ'].fillna(value=0)\n", "slots_df['status'] = df['STATUS']\n", "slots_df.index += cqmid_slot_id_start\n", "slots_df.index.name = 'cqmid_slot_id'\n", "slots_df" ] }, { "cell_type": "code", "execution_count": 26, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
litho_codeexaminer_codesequence_nosubject_codemarks_obtainedmarks_obtained_head_examinerscript_serialscan_serialerror_stringalignment_markscqmid_bundle_idcqmid_slot_idstatuslocal_resolvedglobal_resolvedimage_linkis_editedextra_page_countqr_litho_codeis_litho_mapped
cqmid_record_id
3818731001100100110011011101000100100110200001109011NaN00000000001000000000000000000000001111142114210000100110010011001101110100010010010
381874100101011100000001110000010010011020000210901501610000000002000000000000000000000001111142114210000100101011100000001110000010010010
3818751001101111011110101010011100100110200003109013NaN20000000003000000000000000000000001111142114210000100110111101111010101001110010010
3818761001000100110001101010101101100110200004109019NaN30000000004000000000000000000000001111142114210000100100010011000110101010110110010
3818771001111010001100110011001101100110200005109013NaN40000000005000000000000000000000001111142114210000100111101000110011001100110110010
...............................................................
3834710000100010000011100111111110000021230249109010NaN15980000001599000000000000000000000001111142114240000000010001000001110011111111000000
3834720000101010001110010110011101000021230250109012NaN15990000001600000000000000000000000001111142114240000000010101000111001011001110100000
3834730000010111110010011100111100000021230251109010NaN16000000001601000000000000000000000001111142114240000000001011111001001110011110000000
3834740000010000110010001011100001000021230252109006NaN16010000001602000000000000000000000001111142114240000000001000011001000101110000100000
3834750000111000100011001010000010000021230253109002NaN16020000001603000000000000000000000001111142114240000000011100010001100101000001000000
\n", "

1603 rows × 20 columns

\n", "
" ], "text/plain": [ " litho_code examiner_code sequence_no \\\n", "cqmid_record_id \n", "381873 10011001001100110111010001001001 1020 0001 \n", "381874 10010101110000000111000001001001 1020 0002 \n", "381875 10011011110111101010100111001001 1020 0003 \n", "381876 10010001001100011010101011011001 1020 0004 \n", "381877 10011110100011001100110011011001 1020 0005 \n", "... ... ... ... \n", "383471 00001000100000111001111111100000 2123 0249 \n", "383472 00001010100011100101100111010000 2123 0250 \n", "383473 00000101111100100111001111000000 2123 0251 \n", "383474 00000100001100100010111000010000 2123 0252 \n", "383475 00001110001000110010100000100000 2123 0253 \n", "\n", " subject_code marks_obtained marks_obtained_head_examiner \\\n", "cqmid_record_id \n", "381873 109 011 NaN \n", "381874 109 015 016 \n", "381875 109 013 NaN \n", "381876 109 019 NaN \n", "381877 109 013 NaN \n", "... ... ... ... \n", "383471 109 010 NaN \n", "383472 109 012 NaN \n", "383473 109 010 NaN \n", "383474 109 006 NaN \n", "383475 109 002 NaN \n", "\n", " script_serial scan_serial error_string \\\n", "cqmid_record_id \n", "381873 0 0000000001 00000000000000000000000 \n", "381874 1 0000000002 00000000000000000000000 \n", "381875 2 0000000003 00000000000000000000000 \n", "381876 3 0000000004 00000000000000000000000 \n", "381877 4 0000000005 00000000000000000000000 \n", "... ... ... ... \n", "383471 1598 0000001599 00000000000000000000000 \n", "383472 1599 0000001600 00000000000000000000000 \n", "383473 1600 0000001601 00000000000000000000000 \n", "383474 1601 0000001602 00000000000000000000000 \n", "383475 1602 0000001603 00000000000000000000000 \n", "\n", " alignment_marks cqmid_bundle_id cqmid_slot_id status \\\n", "cqmid_record_id \n", "381873 1111 1421 1421 \n", "381874 1111 1421 1421 \n", "381875 1111 1421 1421 \n", "381876 1111 1421 1421 \n", "381877 1111 1421 1421 \n", "... ... ... ... ... \n", "383471 1111 1421 1424 \n", "383472 1111 1421 1424 \n", "383473 1111 1421 1424 \n", "383474 1111 1421 1424 \n", "383475 1111 1421 1424 \n", "\n", " local_resolved global_resolved image_link is_edited \\\n", "cqmid_record_id \n", "381873 0 0 0 \n", "381874 0 0 0 \n", "381875 0 0 0 \n", "381876 0 0 0 \n", "381877 0 0 0 \n", "... ... ... ... ... \n", "383471 0 0 0 \n", "383472 0 0 0 \n", "383473 0 0 0 \n", "383474 0 0 0 \n", "383475 0 0 0 \n", "\n", " extra_page_count qr_litho_code \\\n", "cqmid_record_id \n", "381873 0 10011001001100110111010001001001 \n", "381874 0 10010101110000000111000001001001 \n", "381875 0 10011011110111101010100111001001 \n", "381876 0 10010001001100011010101011011001 \n", "381877 0 10011110100011001100110011011001 \n", "... ... ... \n", "383471 0 00001000100000111001111111100000 \n", "383472 0 00001010100011100101100111010000 \n", "383473 0 00000101111100100111001111000000 \n", "383474 0 00000100001100100010111000010000 \n", "383475 0 00001110001000110010100000100000 \n", "\n", " is_litho_mapped \n", "cqmid_record_id \n", "381873 0 \n", "381874 0 \n", "381875 0 \n", "381876 0 \n", "381877 0 \n", "... ... \n", "383471 0 \n", "383472 0 \n", "383473 0 \n", "383474 0 \n", "383475 0 \n", "\n", "[1603 rows x 20 columns]" ] }, "execution_count": 26, "metadata": {}, "output_type": "execute_result" } ], "source": [ "records_file = f'{bundle_code}.DAT'\n", "records_df = pd.read_fwf(f'{records_path}/{records_file}', colspecs=[(0,10), (40,72), (72,75), (78,82), (85,89), (97,100), (103,106), (106,110)], header=None, dtype=str, delimiter=\"\\n\\t\").replace(r'^\\s*$', np.nan, regex=True).replace(' ', '0', regex=True)\n", "records_df.columns = ['scan_serial','litho_code','subject_code','examiner_code','sequence_no','marks_obtained','marks_obtained_head_examiner','alignment_marks']\n", "# records_df['exam_type'] = exam_type\n", "records_df['cqmid_bundle_id'] = bundle_df.index[0]\n", "records_df['cqmid_slot_id'] = 0\n", "for i, n, stop in zip(slots_df.index, slots_df['actual_ts'], slots_df['actual_ts'].cumsum()):\n", " records_df.loc[stop-n:stop, 'cqmid_slot_id'] = i\n", "records_df['script_serial'] = range(len(records_df))\n", "records_df['error_string'] = '00000000000000000000000'\n", "records_df['local_resolved'] = 0\n", "records_df['global_resolved'] = 0\n", "records_df['status'] = ''\n", "records_df['is_edited'] = 0\n", "records_df['image_link'] = ''\n", "records_df['extra_page_count'] = 0\n", "records_df['is_litho_mapped'] = 0\n", "records_df['qr_litho_code'] = records_df['litho_code']\n", "records_df.index += cqmid_record_id_start\n", "records_df.index.name = 'cqmid_record_id'\n", "records_df[['litho_code','examiner_code','sequence_no','subject_code','marks_obtained','marks_obtained_head_examiner',\n", "'script_serial','scan_serial','error_string','alignment_marks','cqmid_bundle_id','cqmid_slot_id','status',\n", "'local_resolved','global_resolved','image_link','is_edited','extra_page_count','qr_litho_code','is_litho_mapped']]" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Final Loop" ] }, { "cell_type": "code", "execution_count": 112, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "100%|██████████| 69/69 [00:02<00:00, 31.50it/s]\n" ] } ], "source": [ "bundle_codes = set(f[:-4] for f in os.listdir(slots_path) if f.endswith('.PKT')) & set(f[:-4] for f in os.listdir(records_path) if f.endswith('.DAT'))\n", "\n", "for bundle_code in tqdm(bundle_codes):\n", " bundle_df = pd.DataFrame({}, index=[0])\n", " bundle_df['subject_code'] = bundle_code[1:4]\n", " bundle_df['script_type'] = bundle_code[4:5]\n", " bundle_df['machine_no'] = bundle_code[5:6]\n", " bundle_df['bundle_no'] = bundle_code[6:8]\n", " bundle_df['local_solver_id'] = ''\n", " bundle_df['global_solver_id'] = ''\n", " bundle_df['status'] = 0\n", " bundle_df['exam_id'] = exam_id\n", " bundle_df['no_of_erroneous_script'] = 0\n", " bundle_df['local_start_datetime'] = ''\n", " bundle_df['local_end_datetime'] = ''\n", " bundle_df['solving_order'] = 0\n", " bundle_df.index += cqtop_slot_id_start\n", " bundle_df.index.name = 'cqtop_bundle_id'\n", " # bundle_df.to_sql('CQTopBundles', engine, if_exists='append')\n", "\n", " slots_file = f'{bundle_code}.PKT'\n", " df = pd.DataFrame(DBF(f'{slots_path}/{slots_file}'))\n", " slots_df = pd.DataFrame()\n", " slots_df['center_code'] = df['PKT_NO'].str[0:3]\n", " slots_df = slots_df.join(centers_df, on='center_code')[['center_id']]\n", " slots_df['slot_no'] = df['PKT_NO'].str[3:5]\n", " slots_df['subject_code'] = bundle_df['subject_code']\n", " slots_df['cqtop_bundle_id'] = bundle_df.index[0]\n", " slots_df['total_script'] = df['TS_TOTAL']\n", " slots_df['start_roll'] = ''\n", " slots_df['end_roll'] = df['ENDROLL']\n", " slots_df['actual_ts'] = df['SCANNED']\n", " slots_df['no_of_rejected'] = df['MARK_REJ']\n", " slots_df['status'] = df['STATUS']\n", " slots_df['error_string'] = ''\n", " slots_df.index += cqtop_slot_id_start\n", " slots_df.index.name = 'cqtop_slot_id'\n", " # slots_df.to_sql('CQTopSlots', engine, if_exists='append')\n", "\n", " records_file = f'{bundle_code}.DAT'\n", " records_df = pd.read_fwf(f'{records_path}/{records_file}', colspecs=[(0,10), (41,73), (73,79), (80,90), (91,94), (97,99), (99,103)], header=None, dtype=str, delimiter=\"\\n\\t\").replace(' ', '0', regex=True)\n", " records_df.columns = ['scan_serial','litho_code','roll_no','reg_no','subject_code','extra_page_count','alignment_marks']\n", " records_df['exam_type'] = exam_type\n", " records_df['cqtop_bundle_id'] = bundle_df.index[0]\n", " for i, n, stop in zip(slots_df.index, slots_df['actual_ts'], slots_df['actual_ts'].cumsum()):\n", " records_df.loc[stop-n:stop, 'cqtop_slot_id'] = i\n", " records_df['cqtop_slot_id'] = records_df['cqtop_slot_id'].astype(int)\n", " records_df['script_serial'] = range(len(records_df))\n", " records_df['error_string'] = '00000000000000000000000'\n", " records_df['local_resolved'] = 0\n", " records_df['global_resolved'] = 0\n", " records_df['status'] = ''\n", " records_df['is_edited'] = 0\n", " records_df['image_link'] = ''\n", " records_df['roll_autosolved'] = 0\n", " records_df['subjectcode_autosolved'] = 0\n", " records_df['system_updated_status'] = 0\n", " records_df['responsible_for_duplicate'] = '00000000'\n", " records_df['qr_litho_code'] = records_df['litho_code']\n", " records_df.index += cqtop_record_id_start\n", " records_df.index.name = 'cqtop_record_id'\n", " # records_df.to_sql('CQTopRecords', engine, if_exists='append', chunksize=chunksize, method=method)\n", "\n", " cqtop_bundle_id_start += len(bundle_df)\n", " cqtop_slot_id_start += len(slots_df)\n", " cqtop_record_id_start += len(records_df)" ] } ], "metadata": { "kernelspec": { "display_name": "Python 3.10.6 ('omrproc')", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.10.6" }, "orig_nbformat": 4, "vscode": { "interpreter": { "hash": "407bfa29f0752efef9195014b440e6d1aba1f35e71977bc4a28836510e310ec6" } } }, "nbformat": 4, "nbformat_minor": 2 }