Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Large tables generation speedup #137

Open
wants to merge 2 commits into
base: master
Choose a base branch
from
Open

Large tables generation speedup #137

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
3cad4e5
speed up the generation of large tables
cervinka Nov 21, 2023
6542761
delete unnecessary comments
cervinka Nov 21, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions lib/elixlsx/compiler.ex
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -61,8 +61,8 @@ defmodule Elixlsx.Compiler do

@spec compinfo_from_rows(WorkbookCompInfo.t(), list(list(any()))) :: WorkbookCompInfo.t()
def compinfo_from_rows(wci, rows) do
List.foldl(rows, wci, fn cols, wci ->
List.foldl(cols, wci, fn cell, wci ->
Enum.reduce(rows, wci, fn {_row, cols}, wci ->
Enum.reduce(cols, wci, fn {_col, cell}, wci ->
compinfo_cell_pass(wci, cell)
end)
end)
Expand All @@ -71,7 +71,7 @@ defmodule Elixlsx.Compiler do
@spec compinfo_from_sheets(WorkbookCompInfo.t(), list(Sheet.t())) :: WorkbookCompInfo.t()
def compinfo_from_sheets(wci, sheets) do
List.foldl(sheets, wci, fn sheet, wci ->
compinfo_from_rows(wci, sheet.rows)
compinfo_from_rows(wci, sheet.cells)
end)
end

Expand Down
39 changes: 10 additions & 29 deletions lib/elixlsx/sheet.ex
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ defmodule Elixlsx.Sheet do
cell. See `Font.from_props/1` for a list of options.
"""
defstruct name: "",
rows: [],
cells: %{},
col_widths: %{},
row_heights: %{},
group_cols: [],
Expand All @@ -33,7 +33,7 @@ defmodule Elixlsx.Sheet do

@type t :: %Sheet{
name: String.t(),
rows: list(list(any())),
cells: %{integer() => %{integer() => any()}},
col_widths: %{pos_integer => number},
row_heights: %{pos_integer => number},
group_cols: list(rowcol_group),
Expand Down Expand Up @@ -71,9 +71,13 @@ defmodule Elixlsx.Sheet do
This is mainly used for doctests and does not generate valid CSV (yet).
"""
def to_csv_string(sheet) do
Enum.map_join(sheet.rows, "\n", fn row ->
Enum.map_join(row, ",", fn cell ->
{content, _} = split_cell_content_props(cell)
last_row_idx = Map.keys(sheet.cells) |> Enum.max()

Enum.map_join(0..last_row_idx, "\n", fn row ->
last_col_idx = Map.keys(sheet.cells[row]) |> Enum.max()

Enum.map_join(0..last_col_idx, ",", fn col ->
{content, _} = split_cell_content_props(sheet.cells[row][col])

case content do
nil -> ""
Expand Down Expand Up @@ -119,30 +123,7 @@ defmodule Elixlsx.Sheet do
"""
def set_at(sheet, rowidx, colidx, content, opts \\ [])
when is_number(rowidx) and is_number(colidx) do
cond do
length(sheet.rows) <= rowidx ->
# append new rows, call self again with new sheet
n_new_rows = rowidx - length(sheet.rows)
new_rows = 0..n_new_rows |> Enum.map(fn _ -> [] end)

update_in(sheet.rows, &(&1 ++ new_rows))
|> set_at(rowidx, colidx, content, opts)

length(Enum.at(sheet.rows, rowidx)) <= colidx ->
n_new_cols = colidx - length(Enum.at(sheet.rows, rowidx))
new_cols = 0..n_new_cols |> Enum.map(fn _ -> nil end)
new_row = Enum.at(sheet.rows, rowidx) ++ new_cols

update_in(sheet.rows, &List.replace_at(&1, rowidx, new_row))
|> set_at(rowidx, colidx, content, opts)

true ->
update_in(sheet.rows, fn rows ->
List.update_at(rows, rowidx, fn cols ->
List.replace_at(cols, colidx, [content | opts])
end)
end)
end
put_in(sheet, [Access.key!(:cells), Access.key(rowidx, %{}), colidx], [content | opts])
end

@spec set_col_width(Sheet.t(), String.t(), number) :: Sheet.t()
Expand Down
67 changes: 29 additions & 38 deletions lib/elixlsx/xml_templates.ex
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -90,9 +90,7 @@ defmodule Elixlsx.XMLTemplates do
def make_xl_rel_sheet(sheet_comp_info) do
# I'd love to use string interpolation here, but unfortunately """< is heredoc notation, so i have to use
# string concatenation or escape all the quotes. Choosing the first.
"<Relationship Id=\"#{sheet_comp_info.rId}\" Type=\"http://schemas.openxmlformats.org/officeDocument/2006/relationships/worksheet\" Target=\"worksheets/#{
sheet_comp_info.filename
}\"/>"
"<Relationship Id=\"#{sheet_comp_info.rId}\" Type=\"http://schemas.openxmlformats.org/officeDocument/2006/relationships/worksheet\" Target=\"worksheets/#{sheet_comp_info.filename}\"/>"
end

@spec make_xl_rel_sheets(nonempty_list(SheetCompInfo.t())) :: String.t()
Expand Down Expand Up @@ -124,9 +122,7 @@ defmodule Elixlsx.XMLTemplates do
end

"""
<sheet name="#{xml_escape(sheet_info.name)}" sheetId="#{sheet_comp_info.sheetId}" state="visible" r:id="#{
sheet_comp_info.rId
}"/>
<sheet name="#{xml_escape(sheet_info.name)}" sheetId="#{sheet_comp_info.sheetId}" state="visible" r:id="#{sheet_comp_info.rId}"/>
"""
end

Expand Down Expand Up @@ -216,13 +212,16 @@ defmodule Elixlsx.XMLTemplates do

# TODO i know now about string interpolation, i should probably clean this up. ;)
defp xl_sheet_cols(row, rowidx, wci) do
{updated_row, _id} =
row
|> List.foldl({"", 1}, fn cell, {acc, colidx} ->
updated_row =
Map.keys(row)
|> Enum.sort()
|> Enum.map(&(&1 + 1))
|> Enum.reduce("", fn colidx, acc ->
cell = row[colidx - 1]
{content, styleID, cellstyle} = split_into_content_style(cell, wci)

if is_nil(content) do
{acc, colidx + 1}
acc
else
content =
if CellStyle.is_date?(cellstyle) do
Expand Down Expand Up @@ -275,13 +274,14 @@ defmodule Elixlsx.XMLTemplates do
type ->
"""
<c r="#{U.to_excel_coords(rowidx, colidx)}"

s="#{styleID}" t="#{type}">
<v>#{content_value}</v>
</c>
"""
end

{acc <> cell_xml, colidx + 1}
acc <> cell_xml
end
end)

Expand All @@ -302,9 +302,7 @@ defmodule Elixlsx.XMLTemplates do

defp make_data_validation({start_cell, end_cell, values}) when is_bitstring(values) do
"""
<dataValidation type="list" allowBlank="1" showErrorMessage="1" sqref="#{start_cell}:#{
end_cell
}">
<dataValidation type="list" allowBlank="1" showErrorMessage="1" sqref="#{start_cell}:#{end_cell}">
<formula1>#{values}</formula1>
</dataValidation>
"""
Expand All @@ -319,9 +317,7 @@ defmodule Elixlsx.XMLTemplates do
|> Enum.join("&quot;&amp;&quot;")

"""
<dataValidation type="list" allowBlank="1" showErrorMessage="1" sqref="#{start_cell}:#{
end_cell
}">
<dataValidation type="list" allowBlank="1" showErrorMessage="1" sqref="#{start_cell}:#{end_cell}">
<formula1>&quot;#{joined_values}&quot;</formula1>
</dataValidation>
"""
Expand All @@ -334,32 +330,31 @@ defmodule Elixlsx.XMLTemplates do
defp xl_merge_cells(merge_cells) do
"""
<mergeCells count="#{Enum.count(merge_cells)}">
#{
Enum.map(merge_cells, fn {fromCell, toCell} ->
"<mergeCell ref=\"#{fromCell}:#{toCell}\"/>"
end)
}
#{Enum.map(merge_cells, fn {fromCell, toCell} -> "<mergeCell ref=\"#{fromCell}:#{toCell}\"/>" end)}
</mergeCells>
"""
end

defp xl_sheet_rows(data, row_heights, grouping_info, wci) do
defp xl_sheet_rows(cells, row_heights, grouping_info, wci) do
max_row = Map.keys(cells) |> Enum.max()

rows =
Enum.zip(data, 1..length(data))
|> Enum.map_join(fn {row, rowidx} ->
Map.keys(cells)
|> Enum.sort()
|> Enum.map_join(fn rowidx ->
row = cells[rowidx]

"""
<row r="#{rowidx}" #{get_row_height_attr(row_heights, rowidx)}#{
get_row_grouping_attr(grouping_info, rowidx)
}>
#{xl_sheet_cols(row, rowidx, wci)}
<row r="#{rowidx + 1}" #{get_row_height_attr(row_heights, rowidx + 1)}#{get_row_grouping_attr(grouping_info, rowidx + 1)}>
#{xl_sheet_cols(row, rowidx + 1, wci)}
</row>
"""
end)

if (length(data) + 1) in grouping_info.collapsed_idxs do
if (max_row + 1) in grouping_info.collapsed_idxs do
rows <>
"""
<row r="#{length(data) + 1}" collapsed="1"></row>
<row r="#{max_row + 1}" collapsed="1"></row>
"""
else
rows
Expand Down Expand Up @@ -506,7 +501,7 @@ defmodule Elixlsx.XMLTemplates do
"""
<sheetData>
""" <>
xl_sheet_rows(sheet.rows, sheet.row_heights, grouping_info, wci) <>
xl_sheet_rows(sheet.cells, sheet.row_heights, grouping_info, wci) <>
~S"""
</sheetData>
""" <>
Expand Down Expand Up @@ -554,9 +549,7 @@ defmodule Elixlsx.XMLTemplates do
top_left_cell = U.to_excel_coords(row_idx + 1, col_idx + 1)

{"pane=\"#{pane}\"",
"<pane xSplit=\"#{col_idx}\" ySplit=\"#{row_idx}\" topLeftCell=\"#{top_left_cell}\" activePane=\"#{
pane
}\" state=\"frozen\" />"}
"<pane xSplit=\"#{col_idx}\" ySplit=\"#{row_idx}\" topLeftCell=\"#{top_left_cell}\" activePane=\"#{pane}\" state=\"frozen\" />"}

_any ->
{"", ""}
Expand All @@ -575,9 +568,7 @@ defmodule Elixlsx.XMLTemplates do

"""
<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
<sst xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main" count="#{len}" uniqueCount="#{
len
}">
<sst xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main" count="#{len}" uniqueCount="#{len}">
""" <>
Enum.map_join(stringlist, fn {_, value} ->
# the only two characters that *must* be replaced for safe XML encoding are & and <:
Expand Down