Read/Write a big file in C

Question

I have to do a project where I need to generate the largest number of records of a struct , save them in a file ( I think in binary mode is the best option ) and then reads them . The generated file should have different sizes { 1gb , 10gb , 20gb ...} But how can I allocate memory to access them ? I'm using linked list to store the data , the following lists of code I'm using:

#include "Lista.h"
#include <stdlib.h>
#include <stdio.h>

void inicializa_lista(Lista *l, int t)
{
    l->tamInfo = t;
    l->cabeca = NULL;
}

int insereNoInicio(Lista *l, void *info)
{
    Elemento *p = aloca_elemento(l->tamInfo, info);

    if(p == NULL)
        return 0;/*Erro na alocação.*/

    p->info = malloc(l->tamInfo);

    if(p->info == NULL)
    {
        free(p);
        return 0;/*Erro.*/
    }

    memcpy(p->info, info, l->tamInfo);

    p->proximo = l->cabeca;

    l->cabeca = p;

    return 1;
}

int insereNoFim(Lista *l, void *info)
{
    if(lista_vazia(*l))
        return insereNoInicio(l, info);

    Elemento *p = aloca_elemento(l->tamInfo, info);

    if(p == NULL)
        return 0;

    Elemento *aux = l->cabeca;

    while(aux->proximo != NULL)
        aux = aux->proximo;

    p->proximo = NULL;

    aux->proximo = p;

    return 1;
}

int removeNoInicio(Lista *l, void *info)
{
    if(lista_vazia(*l))
        return ERRO_LISTA_VAAZIA;

    Elemento *p = l->cabeca;

    l->cabeca = p->proximo;/*equivalentes l->cabeca = l->cabeca->proximo;*/

    memcpy(info, p->info, l->tamInfo);

    free(p->info);

    free(p);

    return 1;
}

int removeNoFim(Lista *l, void *info)
{
    if(lista_vazia(*l))
        return ERRO_LISTA_VAAZIA;

    if(l->cabeca->proximo == NULL)/* somente quando a lista tem um elemento */
        return removeNoInicio(l, info);

    Elemento *p = l->cabeca;

    while(p->proximo->proximo != NULL)
        p = p->proximo;

    memcpy(info, p->proximo->info, l->tamInfo);

    free(p->proximo->info);
    free(p->proximo);

    p->proximo = NULL;

    return 1;
}

int lista_vazia(Lista l)
{
    return l.cabeca == NULL;
}

Elemento *aloca_elemento(int tamInfo, void *info)
{
    Elemento *p = malloc(sizeof(Elemento));

    if(p == NULL)
        return NULL;

    p->info = malloc(tamInfo);

    if(p->info == NULL)
    {
        free(p);

        return NULL;
    }

    memcpy(p->info, info, tamInfo);

    return p;
}

void mostra_lista(Lista l, void (*mostra_info)(void *))
{
    if(lista_vazia(l))
        printf("A lista está vazia\n");
    else
    {
        Elemento *p = l.cabeca;

        printf("Dados da Lista:\n");

        while(p != NULL)
        {
            mostra_info(p->info);
            p = p->proximo;
        }
    }
}

void limpa_lista(Lista *l)
{
    Elemento *p = l->cabeca;

    while(p != NULL)
    {
        Elemento *aux = p->proximo;

        free(p->info);
        free(p);

        p = aux;
    }

    l->cabeca = NULL;
}

int insereNaPosicao(Lista *l,void *info,int pos){
    if(pos<0)
        return ERRO_POSICAO_INVALIDA;
    if(pos==0)
        return insereNoFim(l,info);
    Elemento *p=l->cabeca;
    int cont =0;
    while(cont<pos-1 && p->proximo!=NULL){
        p=p->proximo;
        cont++;
    }
    if(cont!=pos-1)
        return ERRO_POSICAO_INVALIDA;
    Elemento *novo = aloca_elemento(l->tamInfo,info);
    if(novo==NULL)
        return 0; // ERRO ALOCACAO
    novo->proximo=p->proximo;
    p->proximo=novo;
    return 1;
}

int removeNaPosicao(Lista *l,void *info,int pos){
    if(lista_vazia(*l)) return ERRO_LISTA_VAAZIA;
    if(pos<0) return ERRO_POSICAO_INVALIDA;
    Elemento *p = l->cabeca;
    if(pos==0){
        removeNoInicio(l,info);
    }
    int cont;
    while(cont<pos-1 &&p->proximo!=NULL){
        p=p->proximo;
        cont++;
    }
    if(cont!=pos-1) return ERRO_POSICAO_INVALIDA;
    Elemento *aux = p->proximo;
    p->proximo = aux ->proximo;
    free(aux->info);
    free(aux);
    return 1;
}


int compara_float(void *a,void *b){
    float *p1=a,*p2=b;
    if(*p1>*p2) return 1;
    if(*p1<*p2) return -1;
    return 0;
}


int insereEmOrdem(Lista *l,void *info,int(*compara)(void*,void*)){
    int cont =0;
    Elemento *p = l->cabeca;
    while(p!=NULL && compara(info,p->info)>0){
        cont++;
        p=p->proximo;
    }
    return insereNaPosicao(l,info,cont);

}

///// HEADER

#define ERRO_LISTA_VAAZIA -1
#define ERRO_POSICAO_INVALIDA -2


typedef struct{
    char* nome;
    int matricula;
    int notas;
    int faltas;
}Diario;

typedef struct ele
{
    void *info;
    struct ele *proximo;
}Elemento;

typedef struct
{
    int tamInfo;
    Elemento *cabeca;
}Lista;

void inicializa_lista(Lista *l, int t);

int insereNoInicio(Lista *l, void *info);

int insereNoFim(Lista *l, void *info);

int removeNoInicio(Lista *l, void *info);

int removeNoFim(Lista *l, void *info);

int lista_vazia(Lista l);

Elemento *aloca_elemento(int tamInfo, void *info);

void mostra_lista(Lista l, void (*mostra_info)(void *));

void limpa_lista(Lista *l);

int insereNaPosicao(Lista *l,void *info,int pos);

int insereEmOrdem(Lista *l,void *info,int(*compara)(void*,void*));

int compara_float(void *a,void *b);

Answer 1

It all depends on how you are planning to use the data. Technically you have the memory between ram and virtual memory. But keep in mind if you do tnis randomly accessing data through you file may (meaning probabably) cause your system to thrash (slow down). You will probabally be able to sequetially access the file.

Some fairly standard techinques: 1) use multiple theads: several accessing the file, one or more processing the data (actor pattern) 2) sorting the data 3) multiple caches

Now, before you implement these, this is exactly what SQL databases a do for you (and several other things). At the very least, a little data modelling (in your case a one table database) will permit you to to test how much your data processing algrithm will be sped up using these techinques.

Ps: I once implemented a tree like structure because I was told to do so. The structure worked well, but it took time to implement. However it would have been faster to do a little data modeling, and work on the algorithm first as we knew the structure would work, the algrithm turned out to not scale well, so the project needed to be refactored a bit. In this, the database is quite a bit easier to change.

Answer 2

Do you have that much RAM? Than don't bother with tricks, just dump the memory into a file and vice versa.

But I bet you don't have that much memory (I'm already see the future reader in a couple of years raising their eyebrow in sheer disbelief). So you need to slice your data in parts, each small enough to fit into a reasonable amount of RAM.

The actual data is in the struct named Diario, is that correct? That makes it the smallest possible slice. The complete content of it can be written as plain text, no need for a binary file, all you need is a simple CSV file.

Writing a CSV is easy, reading is a bit more complex but as you have a fixed format it should be relatively simple.

Searching an entry can mean to search the whole file which can last a while. I would suggest to add some kind of index to decrease the search-time.