/*--------------------------------------------------------------------------
 * database.ox - code for Sample and Database classes
 *
 *       (C) Jurgen Doornik 1994
 *
 *--------------------------------------------------------------------------*/

#include <oxstd.h>
#include <database.h>

enum
{   RS_BINARY_SKIP, RS_ASCII_OBS, RS_ASCII_VAR, RS_BINARY_VAR,
    RS_PRINT_OBS,   RS_FMT_VAR,   RS_XLS_OBS,   RS_WKS_OBS,
    RS_UNKNOWN,     RS_FAILED,    RS_DHT_DAT,   RS_SIZEMAT,
    RS_MAT,         RS_BSIZEMAT,  RS_BMAT,      RS_FMT,
	RS_DTA,			RS_BIN_DAT,	  RS_CSV_OBS
};

loaddata(const file, const type, const cvar, const cobs, const fskip,
    const adata, const anames, const afrequency,
    const ayear1, const aperiod1, const ayear2, const aperiod2);

savedata(const file, const type, const cvar, const cobs, const fappend,
    const mdata, const asnames, const ifrequency,
    const iyear1, const iperiod1, const iyear2, const iperiod2);

getgrouplag(const mdata, const mvarsel, const mlagsel, const mselgroup,
    const t1, const t2, const group, const lag1, const lag2);

findsample(const mdata, const mvarsel, const mlagsel,
    const it1, const it2, const dmisval, const at1, const at2);

const decl DB_MISVAL = .;

Sample::GetFrequency()
{   return m_iFreq;
}
Sample::GetYear1()
{   return m_iYear1;
}
Sample::GetPeriod1()
{   return m_iPeriod1;
}
Sample::GetYear2()
{   return m_iYear2;
}
Sample::GetPeriod2()
{   return m_iPeriod2;
}
Sample::GetSize()
{   return (m_iYear2 - m_iYear1) * m_iFreq + m_iPeriod2 - m_iPeriod1 + 1;
}
Sample::GetIndex(const iYear, const iPeriod)
{   return (iYear - m_iYear1) * m_iFreq + iPeriod - m_iPeriod1;
}
Sample::ObsYear(iObs)
{
    decl nyr = int(iObs / m_iFreq);
    decl nper = m_iPeriod1 + iObs - nyr * m_iFreq;
    if (nper > m_iFreq)  ++nyr;
    else if (nper < 1)  --nyr;

return m_iYear1 + nyr;
}
Sample::ObsPeriod(iObs)
{
    decl nyr = int(iObs / m_iFreq);
    decl nper = m_iPeriod1 + iObs - nyr * m_iFreq;
    if (nper > m_iFreq)  nper -= m_iFreq;
    else if (nper < 1)   nper += m_iFreq;

return nper;
}

Database::Database()
{
    m_mData = new matrix[1][0];
    m_iFreq = m_iYear1 = m_iPeriod1 = m_iYear2 = m_iPeriod2 = 0;
    m_dMisVal = DB_MISVAL;
    DeSelect();
}

Database::Create(const iFreq, const iYear1, const iPeriod1, const iYear2,
    const iPeriod2)
{
    if (m_iFreq)
    {   eprint("Database already created\n");
        return;
    }
    m_iFreq = iFreq;
    m_iYear1 = iYear1;  m_iPeriod1 = iPeriod1;
    m_iYear2 = iYear2;  m_iPeriod2 = iPeriod2;

    decl ct = GetIndex(iYear2, iPeriod2) - GetIndex(iYear1, iPeriod1) + 1;
    if (ct <= 0)
    {   eprint("No observations in sample, cannot create database\n");
        return;
    }
    m_mData = new matrix[ct][0];
    m_asNames = new array[0];
}
Database::Load(const file)
{
    return loaddata(file, RS_UNKNOWN, 0, 0, 0, &m_mData, &m_asNames, &m_iFreq,
        &m_iYear1, &m_iPeriod1, &m_iYear2, &m_iPeriod2);
}
Database::LoadIn7(const file)
{
    return loaddata(file, RS_BINARY_VAR, 0, 0, 0, &m_mData, &m_asNames, &m_iFreq,
        &m_iYear1, &m_iPeriod1, &m_iYear2, &m_iPeriod2);
}
Database::LoadWks(const file)
{
    return loaddata(file, RS_WKS_OBS, 0, 0, 0, &m_mData, &m_asNames, &m_iFreq,
        &m_iYear1, &m_iPeriod1, &m_iYear2, &m_iPeriod2);
}
Database::LoadXls(const file)
{
    return loaddata(file, RS_XLS_OBS, 0, 0, 0, &m_mData, &m_asNames, &m_iFreq,
        &m_iYear1, &m_iPeriod1, &m_iYear2, &m_iPeriod2);
}
Database::LoadFmtVar(const file)
{
    return loaddata(file, RS_FMT_VAR, 0, 0, 0, &m_mData, &m_asNames, &m_iFreq,
        &m_iYear1, &m_iPeriod1, &m_iYear2, &m_iPeriod2);
}
Database::LoadObs(const file, const cVar, const cObs, const iYear1,
    const iPeriod1, const iFreq, const fOffendMis)
{
    m_iYear1 = iYear1;
    m_iPeriod1 = iPeriod1;
    m_iFreq = iFreq;
    return loaddata(file, RS_ASCII_OBS, cVar, cObs, fOffendMis, &m_mData, &m_asNames,
        &m_iFreq, &m_iYear1, &m_iPeriod1, &m_iYear2, &m_iPeriod2);
}
Database::LoadVar(const file, const cVar, const cObs, const iYear1,
    const iPeriod1, const iFreq, const fOffendMis)
{
    m_iYear1 = iYear1;
    m_iPeriod1 = iPeriod1;
    m_iFreq = iFreq;
    return loaddata(file, RS_ASCII_VAR, cVar, cObs, fOffendMis, &m_mData, &m_asNames,
        &m_iFreq, &m_iYear1, &m_iPeriod1, &m_iYear2, &m_iPeriod2);
}
Database::LoadDht(const file, const iYear1, const iPeriod1, const iFreq)
{
    m_iYear1 = iYear1;
    m_iPeriod1 = iPeriod1;
    m_iFreq = iFreq;
    return loaddata(file, RS_DHT_DAT, 0, 0, 0, &m_mData, &m_asNames, &m_iFreq,
        &m_iYear1, &m_iPeriod1, &m_iYear2, &m_iPeriod2);
}
Database::LoadDta(const file, const iYear1, const iPeriod1, const iFreq)
{
    m_iYear1 = iYear1;
    m_iPeriod1 = iPeriod1;
    m_iFreq = iFreq;
    return loaddata(file, RS_DTA, 0, 0, 0, &m_mData, &m_asNames, &m_iFreq,
        &m_iYear1, &m_iPeriod1, &m_iYear2, &m_iPeriod2);
}
Database::SaveIn7(const file)
{
    return savedata(file, RS_BINARY_VAR, columns(m_mData), rows(m_mData), 0, m_mData,
        m_asNames, m_iFreq, m_iYear1, m_iPeriod1, m_iYear2, m_iPeriod2);
}
Database::SaveWks(const file)
{
    return savedata(file, RS_WKS_OBS, columns(m_mData), rows(m_mData), 0, m_mData,
        m_asNames, m_iFreq, m_iYear1, m_iPeriod1, m_iYear2, m_iPeriod2);
}
Database::SaveXls(const file)
{
    return savedata(file, RS_XLS_OBS, columns(m_mData), rows(m_mData), 0, m_mData,
        m_asNames, m_iFreq, m_iYear1, m_iPeriod1, m_iYear2, m_iPeriod2);
}
Database::SaveFmtVar(const file)
{
    return savedata(file, RS_FMT_VAR, columns(m_mData), rows(m_mData), 0, m_mData,
        m_asNames, m_iFreq, m_iYear1, m_iPeriod1, m_iYear2, m_iPeriod2);
}
Database::SaveObs(const file)
{
    return savedata(file, RS_ASCII_OBS, columns(m_mData), rows(m_mData), 0, m_mData,
        m_asNames, m_iFreq, m_iYear1, m_iPeriod1, m_iYear2, m_iPeriod2);
}
Database::SaveVar(const file)
{
    return savedata(file, RS_ASCII_VAR, columns(m_mData), rows(m_mData), 0, m_mData,
        m_asNames, m_iFreq, m_iYear1, m_iPeriod1, m_iYear2, m_iPeriod2);
}
Database::Remove(const sName)
{
    decl j, k, cdb;

    cdb = columns(m_mData);          /* number of variables in the database */

    j = strfind(m_asNames, sName);                /* locate in the database */
    if (j >= 0)
    {   decl asn = {};
		if (j > 0)   	 asn ~= m_asNames[ : j - 1];
		if (j < cdb - 1) asn ~= m_asNames[j + 1 : ];
		m_asNames = asn;
		m_mData = dropc(m_mData, j);
    }
}
Database::RemoveObsIf(const vRemove)
{
    decl vsel, imin, imax;

	if (sizerc(vRemove) != rows(m_mData))
	{
		eprint("RemoveObsIf: vRemove does not match database\n");
	}
	vsel = vecindex(!vRemove);
	if (sizerc(vsel) == 0)
		return;
	imin = min(vsel);
	
	m_mData = m_mData[vsel][];   					 /* reduce the database */

	m_iYear1   = ObsYear(imin);
	m_iPeriod1 = ObsPeriod(imin);
	m_iYear2   = ObsYear(rows(m_mData) - 1);
	m_iPeriod2 = ObsPeriod(rows(m_mData) - 1);
}
Database::Renew(const mNew, const sName, ...)
{
    decl i, j, it1, args = va_arglist();
	it1 = sizeof(args) > 0 ? args[0] : 0;

    j = strfind(m_asNames, sName);                /* locate in the database */
    if (any(j .== -1))
	{
		if (isstring(sName))
			Append(mNew[][0], sName, it1);
		else
			for (i = 0; i < columns(j); ++i)
				if (j[i] == -1)
					Append(mNew[][i], sName[i], it1);
	    j = strfind(m_asNames, sName);         /* re-locate in the database */
    	if (any(j .== -1))
	    {   eprint("Renew(): could not append variable(s)\n");
			return;
		}
	}
    m_mData[it1:][j] = mNew;
}
Database::RenewBlock(const mNew, const iVarIndex)
{
    m_mData[][iVarIndex : ] = mNew;
}
Database::Append(const mNew, const asNew, ...)
{
    decl i, ct, cnew, newdata, it1, args = va_arglist();

	it1 = sizeof(args) > 0 ? args[0] : 0;

    if (m_iFreq == 0)
		Create(1, 1, 1, rows(mNew), 1);

	if (isstring(asNew))
	{
		cnew = 1;
		if (columns(mNew) != 1)
	    {   eprint("Append(): one data column expected\n");
    	    return;
		}
	}
    else if (isarray(asNew))
    {	cnew = columns(mNew);
		if (cnew != sizeof(asNew))
	    {   eprint("Append(): need same number of names as variables\n");
    	    return;
		}
		else if (cnew == 0)
			return;
    }
	else
    {   eprint("Append(): asNew has wrong type\n");
   	    return;
	}
	if (any(strfind(m_asNames, asNew) .>= 0))
    {	eprint("Append() variable(s) already exist(s), use Renew()\n");
   	    return;
	}
	if (rows(m_mData) - it1 < rows(mNew))
    	eprint("Append() warning: cannot grow sample, use Grow()\n");
	
	ct = rows(m_mData);
    newdata = constant(m_dMisVal, ct, cnew);    /* fill with missing values */
    newdata[it1:][] = mNew;           /* bring in line with database sample */

    m_mData ~= newdata;
	m_asNames ~= asNew;
}
Database::Grow(const cTadd)
{
    if (m_iFreq == 0)
		Create(1, 1, 1, cTadd, 1);
	else
    {	m_mData |= constant(m_dMisVal, cTadd, columns(m_mData));
		m_iYear2   = ObsYear(rows(m_mData) - 1);
		m_iPeriod2 = ObsPeriod(rows(m_mData) - 1);
	}
}
Database::Deterministic(const iCseason)
{
    decl i, ct;

    if (m_iFreq == 0)
    {   eprint("Use Create() first\n");
        return;
    }

    ct = rows(m_mData);
	if (strfind(m_asNames, "Constant") == -1)
	{
	    m_mData ~= ones(ct,1);							 /* append constant */
    	m_asNames ~= { "Constant" };
	}
	if (strfind(m_asNames, "Trend") == -1)
	{
	    m_mData ~= range(1,ct)';							/* append trend */
    	m_asNames ~= { "Trend" };
	}
	
    if (m_iFreq == 1 || iCseason < 0 || iCseason > 3)
		return;
		
	decl asseas = {"Season", "CSeason", "Seasonal", "CSeasonal"};
	if (strfind(m_asNames, asseas[iCseason]) != -1)
		return;
    
    decl season, imax = m_iFreq, per = GetPeriod1() - 1;
	if (iCseason == 2 || iCseason == 3) imax = 1;

    for (i = 0; i < imax; i++)
    {
		season = fmod(range(per, per + ct - 1)', m_iFreq) .== i;

        if (iCseason == 1 || iCseason == 3)
            season -= 1 / m_iFreq;
        m_mData ~= season;
        m_asNames ~= (i == 0) ? asseas[iCseason] : sprint(asseas[iCseason], "_", i);
    }
}
Database::GetAll()
{
    return m_mData;
}
Database::GetAllNames()
{
    return m_asNames;
}
Database::GetVar(const sName)
{
    decl j = strfind(m_asNames, sName);           /* locate in the database */
    return m_mData[][deletec(j, -1)];
}
Database::GetVarByIndex(const iVar)
{
    return m_mData[][iVar];
}
Database::GetVarNameByIndex(const iVar)
{
    return m_asNames[iVar];
}
Database::GetVarIndex(const sName)
{
    decl j = strfind(m_asNames, sName);           /* locate in the database */
    return deletec(j, -1);
}
Database::Rename(const sNewName, const sOldName)
{
    decl j = strfind(m_asNames, sOldName);        /* locate in the database */
    if (isstring(sOldName) && j >= 0)
        m_asNames[j] = sNewName;
	else
        eprint("Rename(): variable not found.\n");
}
Database::GetSample()
{
	decl it2 = rows(m_mData) - 1;
	if (m_iFreq == 1)
		return sprint(ObsYear(0), " - ", ObsYear(it2));
	else
		return sprint(ObsYear(0), " (", ObsPeriod(0), ") - ",
    	    ObsYear(it2), " (", ObsPeriod(it2), ")");
}
Database::Info()
{
    decl i, t1, t2, cx, ct, mean, vx;

    if (m_iFreq == 0)
    {   print("\nDatabase not yet created\n");
        return;
    }
    cx = columns(m_mData);
    ct = rows(m_mData);

    if (cx == 0)
    {   print("\nDatabase empty\n");
        return;
    }

    println("\n\n---- Database information ----");
	println("Sample:    ", GetSample()," (", ct," observations)");
    println("Frequency: ", m_iFreq);
    println("Variables: ", cx);

    println("\n", "%-13s", "Variable", "   #obs", "  #miss",
        "         min        mean         max     std.dev");
    for (i = 0; i < cx; i++)
    {
        vx = deleter(m_mData[][i]);
        print("%-13s", m_asNames[i]);
		if (sizerc(vx) == 0)
			println("   No valid observations.");
		else
	        println(" ", "%6d", rows(vx), " ", "%6d", ct - rows(vx), " ",
            	  "%11.5g", double(min(vx)), " ",
	              "%11.5g", double(meanc(vx)), " ",
    	          "%11.5g", double(max(vx)), " ",
        	      "%11.5g", sqrt(double(varc(vx))));
    }
}

//------------------------ selection part ----------------------------------
Database::SelectByIndex(const iGroup, const iVar, const iLag0, const iLag1)
{
	if (sizerc(iVar) == 1)
	{
		if (iVar < 0 || iVar >= columns(m_mData))
			return;
		decl k = iLag1 - iLag0 + 1;	 /* could check for duplicate selection */
    	m_mVarsel ~= constant(iVar, 1, k);
	    m_mLagsel ~= range(iLag0, iLag1);
    	m_mSelgroup ~= constant(iGroup, 1, k);
	}
	else
	{
		decl i, civar, clag0, clag1;
		civar = sizerc(iVar);  clag0 = sizerc(iLag0);  clag1 = sizerc(iLag1);
		if (clag0 > 1 && civar != clag0 || clag1 > 1 && civar != clag1)
		{	eprint("SelectVarByIndex(): argument dimension mismatch\n");
			return;
		}
		for (i = 0; i < civar; ++i)
			SelectByIndex(iGroup[0], iVar[i], iLag0[clag0 > 1 ? i : 0],
				iLag1[clag1 > 1 ? i : 0] );
	}
	m_iT1sel = m_iT2sel = -1;   /* invalidate any previous sample selection */
}

Database::Select(const iGroup, const aSel)
{
    decl i, j, k, carg;

    carg = sizeof(aSel);

    for (i = 0; i < carg; i += 3)            /* "name", first lag, last lag */
    {
        j = strfind(m_asNames, aSel[i]);          /* locate in the database */
        if (j >= 0)
			SelectByIndex(iGroup, j, aSel[i + 1], aSel[i + 2]);
        else
            eprint(aSel[i], " not found in the database\n");
    }
    m_iT1sel = m_iT2sel = -1;   /* invalidate any previous sample selection */
}
Database::DeSelect()
{
    m_mVarsel = m_mLagsel = m_mSelgroup = new matrix[1][0];
    m_iT1sel = m_iT2sel = -1;
}
Database::GetSelInfo()
{
    return { m_mVarsel, m_mSelgroup, m_mLagsel, m_iT1sel, m_iT2sel };
}
Database::SetSelInfo(const asInfo)
{
	[ m_mVarsel, m_mSelgroup, m_mLagsel, m_iT1sel, m_iT2sel] = asInfo;
}
Database::GetGroup(const iGroup)
{
    if (m_iT1sel == -1)
    {   eprint("No database sample selected yet.\n");
        return 0;
    }
    return getgrouplag(m_mData, m_mVarsel, m_mLagsel, m_mSelgroup,
        m_iT1sel, m_iT2sel, iGroup, -10000, 10000);
}
Database::GetGroupLag(const iGroup, const iLag1, const iLag2)
{
    if (m_iT1sel == -1)
    {   eprint("No database sample selected yet.\n");
        return 0;
    }
    return getgrouplag(m_mData, m_mVarsel, m_mLagsel, m_mSelgroup,
        m_iT1sel, m_iT2sel, iGroup, iLag1, iLag2);
}
Database::GetGroupNames(const iGroup, const aasNames)
{
	GetGroupLagNames(iGroup, -10000, 10000, aasNames);
}
Database::GetGroupLagNames(const iGroup, const iLag1, const iLag2, const aasNames)
{
    decl i, as, ilag, csel;

    as = new array[0];
    csel = columns(m_mVarsel);
	
    for (i = 0; i < csel; ++i)
        if (m_mSelgroup[0][i] == iGroup)
		{
			ilag = int(m_mLagsel[0][i]);
			if (ilag >= iLag1 && ilag <= iLag2)
	        {
				if (ilag == 0)
            		as ~= m_asNames[m_mVarsel[0][i]];
				else
					as ~= sprint(m_asNames[m_mVarsel[0][i]], "_", "%d", ilag);
	        }
		}
	aasNames[0] = as;
}

Database::GetMaxLag()
{
    return max(m_mLagsel);
}
Database::GetMaxSelLag()
{
    return max(m_mLagsel);
}
Database::GetMaxGroupLag(const iGroup)
{
    return max(m_mLagsel .* (m_mSelgroup .== iGroup));
}
Database::GetSelStart()
{
    return m_iT1sel;
}
Database::GetSelEnd()
{
    return m_iT2sel;
}
Database::GetSelSample()
{
	if (m_iFreq == 1)
		return sprint(ObsYear(m_iT1sel), " - ",	ObsYear(m_iT2sel));
	else
		return sprint(ObsYear(m_iT1sel), " (", ObsPeriod(m_iT1sel), ") - ",
    	    ObsYear(m_iT2sel), " (", ObsPeriod(m_iT2sel), ")");
}
Database::SetSample(const iYear1, const iPeriod1, const iYear2, const iPeriod2)
{
	return SetSelSample(iYear1, iPeriod1, iYear2, iPeriod2);
}
Database::SetSelSample(const iYear1, const iPeriod1, const iYear2, const iPeriod2)
{
    decl t1, t2;

    t1 = (iYear1 == -1) ? 0 : GetIndex(iYear1, iPeriod1);
    t2 = (iYear2 == -1) ? rows(m_mData) - 1 : GetIndex(iYear2, iPeriod2);

                            /* find sample common to ALL selected variables */
    return findsample(m_mData, m_mVarsel, m_mLagsel, t1, t2, m_dMisVal,
        &m_iT1sel, &m_iT2sel);
}
Database::ForceSelSample(const iYear1, const iPeriod1, const iYear2, const iPeriod2)
{
    decl t1, t2, mxt2 = rows(m_mData) - 1;

    t1 = (iYear1 == -1) ? 0 : GetIndex(iYear1, iPeriod1);
    t2 = (iYear2 == -1) ? mxt2 : GetIndex(iYear2, iPeriod2);
	if (t1 < 0) t1 = 0;
	if (t2 > mxt2) t2 = mxt2;
	if (t1 > t2)
	{
		eprint("invalid sample\n");
    	m_iT1sel = m_iT2sel = -1;
		return 0;
	}
	m_iT1sel = t1;
	m_iT2sel = t2;
    return t2 - t1 + 1;
}
Database::FindSelection(const sVar, const iLag)
{
    decl i, csel;

    csel = columns(m_mVarsel);              /* number of selected variables */
    if (!csel)
    {   eprint("No selection\n");
        return -1;
    }

    for (i = 0; i < csel; ++i)                      /* find equation number */
        if (m_mLagsel[0][i] == iLag && m_asNames[m_mVarsel[0][i]] == sVar)
            return i;

    eprint(sVar, " with lag ", iLag, " not found\n");
return -1;
}
Database::DeSelectByIndex(const iSel)
{
	m_mVarsel = dropc(m_mVarsel, iSel);
    m_mLagsel = dropc(m_mLagsel, iSel);
   	m_mSelgroup = dropc(m_mSelgroup, iSel);
}
Database::DeSelectByName(const sVar, const iGroup, const iLag)
{
    decl i;

    for (i = columns(m_mVarsel) - 1; i >= 0; --i)  /* run through selection */
        if (m_mSelgroup[0][i] == iGroup && m_mLagsel[0][i] == iLag
			&& m_asNames[m_mVarsel[0][i]] == sVar)
			DeSelectByIndex(i);
}
