title: optimizing awk command for large file
body: <p>I have these functions to process a 2GB text file. I'm splitting it into 6 parts for simultaneous processing but it is still taking 4+ hours. </p>
  
  <p>What else can I try make the script faster?</p>
  
  <p>A bit of details:</p>
  
  <ol>
  <li>I feed my input csv into a while loop to be read line by line.</li>
  <li>I grabbed the values from the csv line from 4 fields in the read2col function</li>
  <li>The awk in my mainf function takes the values from read2col and do some arithmetic calculation. I'm rounding the result to 2 decimal places. Then, print the line to a text file.</li>
  </ol>
  
  <p>Sample data:</p>
  
  <pre><code>"111","2018-08-24","01:21","ZZ","AAA","BBB","0","","","ZZ","ZZ111","ZZ110","2018-10-12","07:00","2018-10-12","08:05","2018-10-19","06:30","2018-10-19","09:35","ZZZZ","ZZZZ","A","B","146.00","222.26","76.26","EEE","abc","100.50","45.50","0","E","ESSENTIAL","ESSENTIAL","4","4","7","125","125"
  </code></pre>
  
  <hr>
  
  <p>Script:</p>
  
  <pre><code>read2col()
  {
  is_one_way=$(echo "$line"| awk -F'","' '{print $7}')
  price_outbound=$(echo "$line"| awk -F'","' '{print $30}')
  price_exc=$(echo "$line"| awk -F'","' '{print $25}')
  tax=$(echo "$line"| awk -F'","' '{print $27}')
  price_inc=$(echo "$line"| awk -F'","' '{print $26}')
  }
  
  
  #################################################
  #for each line in the csv
  mainf()
  {
  cd $infarepath
  
  while read -r line; do
          #read the value of csv fields into variables
          read2col
  
          if [[ $is_one_way == 0 ]]; then
                  if [[ $price_outbound &gt; 0 ]]; then
                          #calculate price inc and print the entire line to txt file
                          echo $line | awk -v CONVFMT='%.2f' -v pout=$price_outbound -v tax=$tax -F'","' 'BEGIN {OFS = FS} {$25=pout;$26=(pout+(tax / 2)); print}' &gt;&gt;"$csvsplitfile".tmp
                  else
                          #divide price ecx and inc by 2 if price outbound is not greater than 0
                          echo $line | awk -v CONVFMT='%.2f' -v pexc=$price_exc -v pinc=$price_inc -F'","' 'BEGIN {OFS = FS} {$25=(pexc / 2);$26=(pinc /2); print}' &gt;&gt;"$csvsplitfile".tmp
                  fi
          else
                  echo $line &gt;&gt;"$csvsplitfile".tmp
          fi
  
  done &lt; $csvsplitfile
  }
  </code></pre>