CODEWITHSUNDEEP
pythonpython-3.xcsvfor-loop
Mark Chapel
Mark Chapel

Reputation: 497

Create and write to new csv file during 'for' loop

The code below is intended to create a csv file called 'file-0.csv' and start writing lines by iterating through the for loop until it reaches 100 lines. When the limit is reached, it should stop writing to 'file-0.csv', create 'file-1.csv', and continuing the for loop where it left off, start writing to 'file-1.csv' until it reaches 100 lines, and so on until the for loop is complete.

The actual behavior of the code below (complete, and executable) is that it creates the new files as expected (4 total), but it continues to write all lines to 'file-0'....

##### Python 3.5 #####

import csv

rowCounter     = 0
fileCounter    = 0
List_A         = ['a', 'b', 'c', 'd', 'e', 'f', 'g']
List_B         = ['a', 'b', 'c', 'd', 'e', 'f', 'g']
List_C         = ['a', 'b', 'c', 'd', 'e', 'f', 'g']


def new_file():
    global fileCounter
    fileCounter += 1
    with open('file-' + str(fileCounter) + '.csv', 'w') as csvfile:
        rowWriter = csv.writer(csvfile, delimiter=',', quoting=csv.QUOTE_NONE)

with open('file-' + str(fileCounter) + '.csv', 'w') as csvfile:
    rowWriter = csv.writer(csvfile, delimiter=',', quoting=csv.QUOTE_NONE)

    for word1 in List_A:
        for word2 in List_B:
            for word3 in List_C:
                sentence = word1 + word2 + word3
                rowWriter.writerow ([sentence])

                rowCounter += 1

                if rowCounter == 100:
                    new_file()
                    rowCounter = 0
                else:
                    continue

Same code as above, but heavily commented:

##### Python 3.5 #####

######################
####### Setup ########
######################

### Use the CSV library
import csv

### Initialize counters
rowCounter     = 0
fileCounter    = 0

### Create three lists of 'words'
List_A         = ['a', 'b', 'c', 'd', 'e', 'f', 'g']
List_B         = ['a', 'b', 'c', 'd', 'e', 'f', 'g']
List_C         = ['a', 'b', 'c', 'd', 'e', 'f', 'g']

### Function/subroutine that creates new CSV file with incremented filename
def new_file():
    ### Make the variable 'fileCounter' usable by the function
    global fileCounter
    ### Add 1 to 'fileCounter'
    fileCounter += 1
    ### Create new CSV file using the value of 'fileCounter' as part of the name
    with open('file-' + str(fileCounter) + '.csv', 'w') as csvfile:
        rowWriter = csv.writer(csvfile, delimiter=',', quoting=csv.QUOTE_NONE)


######################
#### Main Program ####
######################

### Create initial CSV file using the value of 'fileCounter' as part of the name
with open('file-' + str(fileCounter) + '.csv', 'w') as csvfile:
    ### Create writer object and define how it should behave
    rowWriter = csv.writer(csvfile, delimiter=',', quoting=csv.QUOTE_NONE)

    ### Create & Write lines ###
    ### Nested 'for' loops to iterate through all combinations of words
    for word1 in List_A:
        for word2 in List_B:
            for word3 in List_C:
                ### Build our 'sentence' from the current iteration
                sentence = word1 + word2 + word3
                ### Write 'sentence' to file
                rowWriter.writerow ([sentence])

                ### Increment row counter
                rowCounter += 1

                ### Check if value of rowCounter is 100 and if so, execute
                ### 'new_file' and reset rowCounter to 0. If not, continue.
                if rowCounter == 100:
                    new_file()
                    rowCounter = 0
                else:
                    continue

I suspect the problem is 'rowWriter' not getting updated or passed back to the main loop properly, but I can't seem to figure out how to do it (and anyway, I'm not even sure if that's it).

I've tried to document and make the code "generic" so others can get some use out of any answers. Any help is greatly appreciated.

Upvotes: 2

Views: 5924

Answers (2)

desiato
desiato

Reputation: 1152

Leaving the with block closes with file. Therefore, the new_file function just opens and immediately closes a file.

You could do somthing like the following:

import csv

rowCounter     = 0
fileCounter    = 0
List_A         = ['a', 'b', 'c', 'd', 'e', 'f', 'g']
List_B         = ['a', 'b', 'c', 'd', 'e', 'f', 'g']
List_C         = ['a', 'b', 'c', 'd', 'e', 'f', 'g']

# create file handle
csvfile = open('file-' + str(fileCounter) + '.csv', 'w')

rowWriter = csv.writer(csvfile, delimiter=',', quoting=csv.QUOTE_NONE)

for word1 in List_A:
    for word2 in List_B:
        for word3 in List_C:
            sentence = word1 + word2 + word3
            rowWriter.writerow ([sentence])

            rowCounter += 1

            if rowCounter == 100:
                # close current filehandle
                csvfile.close()
                fileCounter += 1
                # open new file
                csvfile =  open('file-' + str(fileCounter) + '.csv', 'w')
                rowWriter = csv.writer(csvfile, delimiter=',', quoting=csv.QUOTE_NONE)
                rowCounter = 0

# close file
csvfile.close()

or with defining a function:

import csv

rowCounter     = 0
fileCounter    = 0
List_A         = ['a', 'b', 'c', 'd', 'e', 'f', 'g']
List_B         = ['a', 'b', 'c', 'd', 'e', 'f', 'g']
List_C         = ['a', 'b', 'c', 'd', 'e', 'f', 'g']

def new_writer( csvfile, counter ):
     if csvfile: 
         csvfile.close()

     # open new file
     csvfile =  open('file-' + str(counter) + '.csv', 'w')
     rowWriter = csv.writer(csvfile, delimiter=',', quoting=csv.QUOTE_NONE)

     counter += 1

     return rowWriter,csvfile,counter

rowWriter, csvFile, fileCounter = new_writer( None, fileCounter )

for word1 in List_A:
    for word2 in List_B:
        for word3 in List_C:
            sentence = word1 + word2 + word3
            rowWriter.writerow ([sentence])

            rowCounter += 1

            if rowCounter == 100:
                # close current file and open a new one
                rowWriter, csvfile, counter =  new_writer( csvfile, fileCounter )         
                rowCounter = 0

# close file
csvFile.close()

Upvotes: 2

Mark Chapel
Mark Chapel

Reputation: 497

Thanks @desiato!

I accepted your answer, but ended up using lines 23-29 of your code and ended up with this (it works great!):

import csv

rowCounter     = 0
fileCounter    = 0
List_A         = ['a', 'b', 'c', 'd', 'e', 'f', 'g']
List_B         = ['a', 'b', 'c', 'd', 'e', 'f', 'g']
List_C         = ['a', 'b', 'c', 'd', 'e', 'f', 'g']

with open('file-' + str(fileCounter) + '.csv', 'w') as csvfile:
    rowWriter = csv.writer(csvfile, delimiter=',', quoting=csv.QUOTE_NONE)

    for word1 in List_A:
        for word2 in List_B:
            for word3 in List_C:
                sentence = word1 + word2 + word3
                rowWriter.writerow ([sentence])

                rowCounter += 1

                if rowCounter == 100:
                    csvfile.close()
                    fileCounter += 1
                    csvfile =  open('file-' + str(fileCounter) + '.csv', 'w')
                    rowWriter = csv.writer(csvfile, delimiter=',', quoting=csv.QUOTE_NONE)
                    rowCounter = 0
                else:
                    continue

Upvotes: 0

Related Questions