goldenChat base source add
This commit is contained in:
+145
@@ -0,0 +1,145 @@
|
||||
/*
|
||||
* SPDX-License-Identifier: MIT
|
||||
*/
|
||||
package ta4jexamples.datasources;
|
||||
|
||||
import org.ta4j.core.BarSeries;
|
||||
|
||||
import java.io.InputStream;
|
||||
import java.time.Duration;
|
||||
import java.time.Instant;
|
||||
|
||||
/**
|
||||
* Common interface for data sources that load BarSeries using business domain
|
||||
* concepts (ticker, interval, date range).
|
||||
* <p>
|
||||
* This interface abstracts away implementation details (files, APIs, databases)
|
||||
* and allows users to work with trading domain concepts. File-based
|
||||
* implementations will search for files matching the ticker/interval/date range
|
||||
* criteria, while API-based implementations will fetch data from remote
|
||||
* sources.
|
||||
* <p>
|
||||
* <strong>Domain-Driven Usage:</strong>
|
||||
*
|
||||
* <pre>
|
||||
* // All data sources work with business concepts
|
||||
* BarSeriesDataSource yahoo = new YahooFinanceHttpBarSeriesDataSource(true);
|
||||
* BarSeriesDataSource csv = new CsvFileBarSeriesDataSource();
|
||||
* BarSeriesDataSource json = new JsonFileBarSeriesDataSource();
|
||||
*
|
||||
* // Same interface, different implementations
|
||||
* BarSeries aapl = yahoo.loadSeries("AAPL", Duration.ofDays(1), Instant.parse("2023-01-01T00:00:00Z"),
|
||||
* Instant.parse("2023-12-31T23:59:59Z"));
|
||||
*
|
||||
* BarSeries btc = csv.loadSeries("BTC-USD", Duration.ofDays(1), Instant.parse("2023-01-01T00:00:00Z"),
|
||||
* Instant.parse("2023-12-31T23:59:59Z"));
|
||||
* </pre>
|
||||
* <p>
|
||||
* <strong>Direct Source Loading:</strong>
|
||||
* <p>
|
||||
* For cases where you know the exact source identifier (filename, URL, etc.),
|
||||
* use {@link #loadSeries(String)}:
|
||||
*
|
||||
* <pre>
|
||||
* // Direct filename loading (bypasses search logic)
|
||||
* BarSeries series = csv.loadSeries("AAPL-PT1D-20130102_20131231.csv");
|
||||
* </pre>
|
||||
* <p>
|
||||
* <strong>Implementation Behavior:</strong>
|
||||
* <ul>
|
||||
* <li><strong>File-based sources</strong> (CsvFileBarSeriesDataSource,
|
||||
* JsonFileBarSeriesDataSource, BitStampCsvTradesFileBarSeriesDataSource):
|
||||
* Search for files matching ticker/interval/date range patterns in the
|
||||
* classpath or configured directories. The exact filename pattern is
|
||||
* implementation-specific.</li>
|
||||
* <li><strong>API-based sources</strong> (YahooFinanceHttpBarSeriesDataSource):
|
||||
* Fetch data from the API. If caching is enabled, will first check for cached
|
||||
* files matching the criteria.</li>
|
||||
* </ul>
|
||||
*
|
||||
* @since 0.20
|
||||
*/
|
||||
public interface BarSeriesDataSource {
|
||||
|
||||
/**
|
||||
* Loads a BarSeries using business domain concepts.
|
||||
* <p>
|
||||
* This is the primary method for loading data. Implementations interpret the
|
||||
* parameters according to their capabilities:
|
||||
* <ul>
|
||||
* <li><strong>File-based sources</strong>: Search for files matching the
|
||||
* ticker, interval, and date range. The search pattern is
|
||||
* implementation-specific (e.g.,
|
||||
* "{ticker}_bars_from_{startDate}_{endDate}.csv").</li>
|
||||
* <li><strong>API-based sources</strong>: Fetch data from the API for the
|
||||
* specified ticker, interval, and date range. May check cache first if caching
|
||||
* is enabled.</li>
|
||||
* </ul>
|
||||
*
|
||||
* @param ticker the ticker symbol or identifier (e.g., "AAPL", "BTC-USD",
|
||||
* "MSFT")
|
||||
* @param interval the bar interval (e.g., Duration.ofDays(1) for daily bars,
|
||||
* Duration.ofHours(1) for hourly bars)
|
||||
* @param start the start date/time for the data range (inclusive)
|
||||
* @param end the end date/time for the data range (inclusive)
|
||||
* @return a BarSeries containing the loaded data, or null if no matching data
|
||||
* is found or loading fails
|
||||
* @throws IllegalArgumentException if any parameter is invalid (e.g., null
|
||||
* ticker, negative interval, start after end)
|
||||
*/
|
||||
BarSeries loadSeries(String ticker, Duration interval, Instant start, Instant end);
|
||||
|
||||
/**
|
||||
* Loads a BarSeries directly from a known source identifier.
|
||||
* <p>
|
||||
* This method bypasses the search/fetch logic and loads directly from the
|
||||
* specified source. Use this when you know the exact source identifier (e.g.,
|
||||
* filename, URL, resource name).
|
||||
* <p>
|
||||
* For file-based sources, this is typically a filename or resource path. For
|
||||
* API-based sources, this might be a cached file path or a direct API endpoint.
|
||||
*
|
||||
* @param source the source identifier (filename, resource name, URL, etc.)
|
||||
* @return a BarSeries containing the loaded data, or null if loading fails
|
||||
* @throws IllegalArgumentException if the source parameter is invalid or
|
||||
* unsupported
|
||||
*/
|
||||
BarSeries loadSeries(String source);
|
||||
|
||||
/**
|
||||
* Loads a BarSeries from an InputStream.
|
||||
* <p>
|
||||
* This method is optional - implementations that don't support InputStream
|
||||
* loading should throw {@link UnsupportedOperationException}.
|
||||
* <p>
|
||||
* The caller is responsible for closing the InputStream after this method
|
||||
* returns. Implementations should not close the stream unless they fully
|
||||
* consume it.
|
||||
*
|
||||
* @param inputStream the input stream containing the data
|
||||
* @return a BarSeries containing the loaded data, or null if loading fails
|
||||
* @throws UnsupportedOperationException if this data source doesn't support
|
||||
* InputStream loading
|
||||
* @throws IllegalArgumentException if the inputStream is null or invalid
|
||||
*/
|
||||
default BarSeries loadSeries(InputStream inputStream) {
|
||||
throw new UnsupportedOperationException(
|
||||
"InputStream loading not supported by " + this.getClass().getSimpleName());
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the source name for this data source. This is used for building file
|
||||
* search patterns, cache file names, and other source-specific identifiers.
|
||||
* <p>
|
||||
* For example, a Yahoo Finance data source would return "YahooFinance", which
|
||||
* would be used to build cache file names like "YahooFinance-AAPL-1d-...".
|
||||
* <p>
|
||||
* File-based sources that don't use a source prefix (e.g., generic CSV files)
|
||||
* should return an empty string.
|
||||
*
|
||||
* @return the source name, or an empty string if no source prefix is used
|
||||
*/
|
||||
default String getSourceName() {
|
||||
return "";
|
||||
}
|
||||
}
|
||||
+315
@@ -0,0 +1,315 @@
|
||||
/*
|
||||
* SPDX-License-Identifier: MIT
|
||||
*/
|
||||
package ta4jexamples.datasources;
|
||||
|
||||
import org.apache.logging.log4j.LogManager;
|
||||
import org.apache.logging.log4j.Logger;
|
||||
import org.ta4j.core.BarSeries;
|
||||
import org.ta4j.core.BaseBarSeriesBuilder;
|
||||
import org.ta4j.core.num.Num;
|
||||
import ta4jexamples.datasources.file.AbstractFileBarSeriesDataSource;
|
||||
|
||||
import java.io.InputStream;
|
||||
import java.io.InputStreamReader;
|
||||
import java.time.Duration;
|
||||
import java.time.Instant;
|
||||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
import java.util.ListIterator;
|
||||
|
||||
/**
|
||||
* This class builds a Ta4j bar series from a Bitstamp CSV file containing
|
||||
* trades. It reads trade-level data (timestamp, price, volume) and aggregates
|
||||
* them into OHLCV bars suitable for technical analysis.
|
||||
* <p>
|
||||
* Implements {@link BarSeriesDataSource} to support domain-driven loading by
|
||||
* ticker, interval, and date range. Searches for Bitstamp CSV files matching
|
||||
* the specified criteria in the classpath.
|
||||
*/
|
||||
public class BitStampCsvTradesFileBarSeriesDataSource extends AbstractFileBarSeriesDataSource {
|
||||
|
||||
private static final Logger LOG = LogManager.getLogger(BitStampCsvTradesFileBarSeriesDataSource.class);
|
||||
|
||||
private static final String DEFAULT_BITSTAMP_FILE = "Bitstamp-BTC-USD-PT5M-20131125_20131201.csv";
|
||||
|
||||
/**
|
||||
* Creates a new BitStampCsvTradesFileBarSeriesDataSource with "Bitstamp" as the
|
||||
* source name.
|
||||
*/
|
||||
public BitStampCsvTradesFileBarSeriesDataSource() {
|
||||
super("Bitstamp");
|
||||
}
|
||||
|
||||
@Override
|
||||
protected String getFileExtension() {
|
||||
return "csv";
|
||||
}
|
||||
|
||||
@Override
|
||||
protected BarSeries searchAndLoadFile(String ticker, String intervalStr, String sourcePrefix,
|
||||
String startDateTimeStr, String endDateTimeStr, String startDateStr, String endDateStr, Duration interval,
|
||||
Instant start, Instant end) {
|
||||
// Try exact pattern with interval-appropriate format:
|
||||
// {sourceName}-{ticker}-{interval}-{startDateTime}_{endDateTime}.csv
|
||||
String exactPattern = sourcePrefix + ticker.toUpperCase() + "-" + intervalStr + "-" + startDateTimeStr + "_"
|
||||
+ endDateTimeStr + ".csv";
|
||||
BarSeries series = loadBitstampSeries(exactPattern);
|
||||
if (series != null && !series.isEmpty()) {
|
||||
return filterAndAggregateSeries(series, interval, start, end);
|
||||
}
|
||||
|
||||
// Fallback to date-only format for backward compatibility with existing files
|
||||
String exactPatternDateOnly = sourcePrefix + ticker.toUpperCase() + "-" + intervalStr + "-" + startDateStr + "_"
|
||||
+ endDateStr + ".csv";
|
||||
series = loadBitstampSeries(exactPatternDateOnly);
|
||||
if (series != null && !series.isEmpty()) {
|
||||
return filterAndAggregateSeries(series, interval, start, end);
|
||||
}
|
||||
|
||||
// Try broader pattern: {sourceName}-{ticker}-*-{startDateTime}_*.csv
|
||||
String broaderPattern = sourcePrefix + ticker.toUpperCase() + "-*-" + startDateTimeStr + "_*.csv";
|
||||
series = searchAndLoadBitstampFile(broaderPattern, interval, start, end);
|
||||
if (series != null && !series.isEmpty()) {
|
||||
return series;
|
||||
}
|
||||
|
||||
// Fallback to date-only format for broader pattern
|
||||
String broaderPatternDateOnly = sourcePrefix + ticker.toUpperCase() + "-*-" + startDateStr + "_*.csv";
|
||||
series = searchAndLoadBitstampFile(broaderPatternDateOnly, interval, start, end);
|
||||
if (series != null && !series.isEmpty()) {
|
||||
return series;
|
||||
}
|
||||
|
||||
// Try even broader: {sourceName}-{ticker}-*.csv (then filter by date range)
|
||||
String broadestPattern = sourcePrefix + ticker.toUpperCase() + "-*.csv";
|
||||
series = searchAndLoadBitstampFile(broadestPattern, interval, start, end);
|
||||
if (series != null && !series.isEmpty()) {
|
||||
return series;
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public BarSeries loadSeries(String source) {
|
||||
if (source == null || source.trim().isEmpty()) {
|
||||
throw new IllegalArgumentException("Source cannot be null or empty");
|
||||
}
|
||||
return loadBitstampSeries(source);
|
||||
}
|
||||
|
||||
/**
|
||||
* Searches for a Bitstamp CSV file matching the pattern and loads it if found.
|
||||
*
|
||||
* @param pattern the filename pattern to search for (supports wildcards)
|
||||
* @param interval the desired bar interval
|
||||
* @param start the start date
|
||||
* @param end the end date
|
||||
* @return the loaded and filtered BarSeries, or null if not found
|
||||
*/
|
||||
private BarSeries searchAndLoadBitstampFile(String pattern, Duration interval, Instant start, Instant end) {
|
||||
// Try direct pattern match as resource
|
||||
if (!pattern.contains("*")) {
|
||||
BarSeries series = loadBitstampSeries(pattern);
|
||||
if (series != null && !series.isEmpty()) {
|
||||
return filterAndAggregateSeries(series, interval, start, end);
|
||||
}
|
||||
}
|
||||
|
||||
// For wildcard patterns, try common variations
|
||||
String[] variations = { pattern.replace("*", "PT5M"), pattern.replace("*", "PT1D") };
|
||||
for (String variation : variations) {
|
||||
BarSeries series = loadBitstampSeries(variation);
|
||||
if (series != null && !series.isEmpty()) {
|
||||
return filterAndAggregateSeries(series, interval, start, end);
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Filters a series to the date range and re-aggregates with the specified
|
||||
* interval.
|
||||
*
|
||||
* @param series the series to filter and aggregate
|
||||
* @param interval the desired bar interval
|
||||
* @param start the start date (inclusive)
|
||||
* @param end the end date (inclusive)
|
||||
* @return a new BarSeries with bars within the date range and specified
|
||||
* interval, or null if no bars match
|
||||
*/
|
||||
private BarSeries filterAndAggregateSeries(BarSeries series, Duration interval, Instant start, Instant end) {
|
||||
if (series == null || series.isEmpty()) {
|
||||
return null;
|
||||
}
|
||||
|
||||
// Filter trades within date range and re-aggregate with new interval
|
||||
// This is a simplified implementation - in practice, you'd want to
|
||||
// re-aggregate from the original trade data
|
||||
var filteredSeries = new BaseBarSeriesBuilder().withName(series.getName()).build();
|
||||
int barsInDateRange = 0;
|
||||
int barsWithMatchingInterval = 0;
|
||||
Duration actualInterval = null;
|
||||
|
||||
for (int i = 0; i < series.getBarCount(); i++) {
|
||||
var bar = series.getBar(i);
|
||||
Instant barEnd = bar.getEndTime();
|
||||
if (!barEnd.isBefore(start) && !barEnd.isAfter(end)) {
|
||||
barsInDateRange++;
|
||||
if (actualInterval == null) {
|
||||
actualInterval = bar.getTimePeriod();
|
||||
}
|
||||
// If interval matches, add as-is; otherwise would need re-aggregation
|
||||
if (bar.getTimePeriod().equals(interval)) {
|
||||
barsWithMatchingInterval++;
|
||||
filteredSeries.addBar(bar);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Log warning if bars exist in date range but intervals don't match
|
||||
if (barsInDateRange > 0 && barsWithMatchingInterval == 0 && actualInterval != null) {
|
||||
LOG.warn(
|
||||
"Found {} bars within date range [{} to {}], but bar interval ({}) does not match requested interval ({}). "
|
||||
+ "Re-aggregation from original trade data is required but not implemented. Returning null.",
|
||||
barsInDateRange, start, end, actualInterval, interval);
|
||||
}
|
||||
|
||||
return filteredSeries.isEmpty() ? null : filteredSeries;
|
||||
}
|
||||
|
||||
/**
|
||||
* Loads a bar series from the default Bitstamp CSV file. The method reads trade
|
||||
* data from a CSV file containing Bitstamp exchange trades and converts it into
|
||||
* a bar series format suitable for technical analysis.
|
||||
*
|
||||
* @return the bar series from Bitstamp (bitcoin exchange) trades
|
||||
*/
|
||||
public static BarSeries loadBitstampSeries() {
|
||||
return loadBitstampSeries(DEFAULT_BITSTAMP_FILE);
|
||||
}
|
||||
|
||||
/**
|
||||
* Loads a bar series from a specified Bitstamp CSV file. The method reads trade
|
||||
* data from a CSV file containing Bitstamp exchange trades and converts it into
|
||||
* a bar series format suitable for technical analysis.
|
||||
*
|
||||
* @param bitstampCsvFile the path to the CSV file containing Bitstamp trade
|
||||
* data
|
||||
* @return the bar series built from the Bitstamp trades data
|
||||
*/
|
||||
public static BarSeries loadBitstampSeries(String bitstampCsvFile) {
|
||||
|
||||
// Reading all lines of the CSV file
|
||||
InputStream stream = BitStampCsvTradesFileBarSeriesDataSource.class.getClassLoader()
|
||||
.getResourceAsStream(bitstampCsvFile);
|
||||
List<String[]> lines = null;
|
||||
if (stream == null) {
|
||||
LOG.debug("CSV file not found in classpath: {}", bitstampCsvFile);
|
||||
return null;
|
||||
}
|
||||
try (final var csvReader = new com.opencsv.CSVReader(new InputStreamReader(stream))) {
|
||||
lines = csvReader.readAll();
|
||||
lines.remove(0); // Removing header line
|
||||
} catch (Exception ioe) {
|
||||
LOG.error("Unable to load trades from CSV", ioe);
|
||||
}
|
||||
|
||||
var series = new BaseBarSeriesBuilder().withName(bitstampCsvFile).build();
|
||||
if ((lines != null) && !lines.isEmpty()) {
|
||||
|
||||
// Getting the first and last trades timestamps
|
||||
Instant beginTime = null;
|
||||
Instant endTime = null;
|
||||
try {
|
||||
beginTime = Instant.ofEpochMilli(Long.parseLong(lines.get(0)[0]) * 1000);
|
||||
endTime = Instant.ofEpochMilli(Long.parseLong(lines.get(lines.size() - 1)[0]) * 1000);
|
||||
} catch (NumberFormatException nfe) {
|
||||
LOG.error("Invalid trade timestamp format in CSV: {}", nfe.getMessage());
|
||||
return null;
|
||||
}
|
||||
if (beginTime.isAfter(endTime)) {
|
||||
beginTime = endTime;
|
||||
// Since the CSV file has the most recent trades at the top of the file, we'll
|
||||
// reverse the list to feed
|
||||
// the List<Bar> correctly.
|
||||
Collections.reverse(lines);
|
||||
}
|
||||
// build the list of populated bars (default 5-minute bars)
|
||||
buildSeries(series, beginTime, endTime, 300, lines);
|
||||
}
|
||||
|
||||
return series.isEmpty() ? null : series;
|
||||
}
|
||||
|
||||
/**
|
||||
* Builds a list of populated bars from csv data.
|
||||
*
|
||||
* @param beginTime the begin time of the whole period
|
||||
* @param endTime the end time of the whole period
|
||||
* @param duration the bar duration (in seconds)
|
||||
* @param lines the csv data returned by CSVReader.readAll()
|
||||
*/
|
||||
private static void buildSeries(BarSeries series, Instant beginTime, Instant endTime, int duration,
|
||||
List<String[]> lines) {
|
||||
|
||||
Duration barDuration = Duration.ofSeconds(duration);
|
||||
Instant barEndTime = beginTime;
|
||||
ListIterator<String[]> iterator = lines.listIterator();
|
||||
// line number of trade data
|
||||
do {
|
||||
// build a bar
|
||||
barEndTime = barEndTime.plus(barDuration);
|
||||
var bar = series.barBuilder().timePeriod(barDuration).endTime(barEndTime).volume(0).amount(0).build();
|
||||
do {
|
||||
// get a trade
|
||||
String[] tradeLine = iterator.next();
|
||||
Instant tradeTimeStamp;
|
||||
try {
|
||||
tradeTimeStamp = Instant.ofEpochMilli(Long.parseLong(tradeLine[0]) * 1000);
|
||||
} catch (NumberFormatException nfe) {
|
||||
LOG.warn("Invalid trade timestamp format in CSV line, skipping trade: {}",
|
||||
tradeLine.length > 0 ? tradeLine[0] : "empty line", nfe);
|
||||
continue;
|
||||
}
|
||||
// if the trade happened during the bar
|
||||
if (bar.inPeriod(tradeTimeStamp)) {
|
||||
// add the trade to the bar
|
||||
Num tradePrice;
|
||||
Num tradeVolume;
|
||||
try {
|
||||
tradePrice = series.numFactory().numOf(Double.parseDouble(tradeLine[1]));
|
||||
tradeVolume = series.numFactory().numOf(Double.parseDouble(tradeLine[2]));
|
||||
} catch (NumberFormatException nfe) {
|
||||
LOG.warn(
|
||||
"Invalid trade price or volume format in CSV line, skipping trade: price={}, volume={}",
|
||||
tradeLine.length > 1 ? tradeLine[1] : "missing",
|
||||
tradeLine.length > 2 ? tradeLine[2] : "missing", nfe);
|
||||
continue;
|
||||
}
|
||||
bar.addTrade(tradeVolume, tradePrice);
|
||||
} else {
|
||||
// the trade happened after the end of the bar
|
||||
// go to the next bar but stay with the same trade (don't increment i)
|
||||
// this break will drop us after the inner "while", skipping the increment
|
||||
break;
|
||||
}
|
||||
} while (iterator.hasNext());
|
||||
// if the bar has any trades add it to the bars list
|
||||
// this is where the break drops to
|
||||
if (bar.getTrades() > 0) {
|
||||
series.addBar(bar);
|
||||
}
|
||||
} while (barEndTime.isBefore(endTime));
|
||||
}
|
||||
|
||||
public static void main(String[] args) {
|
||||
BarSeries series = BitStampCsvTradesFileBarSeriesDataSource.loadBitstampSeries();
|
||||
|
||||
LOG.debug("Series: {} ({})", series.getName(), series.getSeriesPeriodDescription());
|
||||
LOG.debug("Number of bars: {}", series.getBarCount());
|
||||
LOG.debug("First bar: \n\tVolume: {}\n\tNumber of trades: {}\n\tClose price: {}", series.getBar(0).getVolume(),
|
||||
series.getBar(0).getTrades(), series.getBar(0).getClosePrice());
|
||||
}
|
||||
}
|
||||
+867
@@ -0,0 +1,867 @@
|
||||
/*
|
||||
* SPDX-License-Identifier: MIT
|
||||
*/
|
||||
package ta4jexamples.datasources;
|
||||
|
||||
import com.google.gson.JsonObject;
|
||||
import com.google.gson.JsonParser;
|
||||
import org.apache.logging.log4j.LogManager;
|
||||
import org.apache.logging.log4j.Logger;
|
||||
import org.ta4j.core.Bar;
|
||||
import org.ta4j.core.BarSeries;
|
||||
import org.ta4j.core.BaseBarSeriesBuilder;
|
||||
import ta4jexamples.datasources.http.AbstractHttpBarSeriesDataSource;
|
||||
import ta4jexamples.datasources.http.DefaultHttpClientWrapper;
|
||||
import ta4jexamples.datasources.http.HttpClientWrapper;
|
||||
import ta4jexamples.datasources.http.HttpResponseWrapper;
|
||||
import ta4jexamples.datasources.json.AdaptiveBarSeriesTypeAdapter;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.net.URI;
|
||||
import java.net.URLEncoder;
|
||||
import java.net.http.HttpClient;
|
||||
import java.net.http.HttpRequest;
|
||||
import java.net.http.HttpResponse;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.nio.file.Paths;
|
||||
import java.time.Duration;
|
||||
import java.time.Instant;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.TreeMap;
|
||||
|
||||
/**
|
||||
* Loads OHLCV data from Coinbase Advanced Trade API.
|
||||
* <p>
|
||||
* This loader fetches historical price data from Coinbase's public market data
|
||||
* API without requiring authentication. It supports all Coinbase trading pairs
|
||||
* (e.g., BTC-USD, ETH-USD).
|
||||
* <p>
|
||||
* <strong>Example usage:</strong>
|
||||
*
|
||||
* <pre>
|
||||
* // Load 1 year of daily data for Bitcoin (using days)
|
||||
* BarSeries series = CoinbaseHttpBarSeriesDataSource.loadSeries("BTC-USD", 365);
|
||||
*
|
||||
* // Load 500 bars of hourly data for Ethereum (using bar count)
|
||||
* BarSeries ethSeries = CoinbaseHttpBarSeriesDataSource.loadSeries("ETH-USD", CoinbaseInterval.ONE_HOUR, 500);
|
||||
*
|
||||
* // Load data for a specific date range
|
||||
* Instant start = Instant.parse("2023-01-01T00:00:00Z");
|
||||
* Instant end = Instant.parse("2023-12-31T23:59:59Z");
|
||||
* BarSeries btcSeries = CoinbaseHttpBarSeriesDataSource.loadSeries("BTC-USD", CoinbaseInterval.ONE_DAY, start, end);
|
||||
* </pre>
|
||||
* <p>
|
||||
* <strong>Response Caching:</strong> To enable response caching for faster
|
||||
* subsequent requests, use the constructor with {@code enableResponseCaching}:
|
||||
*
|
||||
* <pre>
|
||||
* CoinbaseHttpBarSeriesDataSource loader = new CoinbaseHttpBarSeriesDataSource(true);
|
||||
* BarSeries series = loader.loadSeriesInstance("BTC-USD", CoinbaseInterval.ONE_DAY, start, end);
|
||||
* </pre>
|
||||
* <p>
|
||||
* To use a custom cache directory, use the constructor with
|
||||
* {@code responseCacheDir}:
|
||||
*
|
||||
* <pre>
|
||||
* CoinbaseHttpBarSeriesDataSource loader = new CoinbaseHttpBarSeriesDataSource("/path/to/cache");
|
||||
* BarSeries series = loader.loadSeriesInstance("BTC-USD", CoinbaseInterval.ONE_DAY, start, end);
|
||||
* </pre>
|
||||
* <p>
|
||||
* When caching is enabled, responses are saved to the cache directory (default:
|
||||
* {@code temp/responses}) and reused for requests within the cache validity
|
||||
* period (based on the interval). For example, daily data is cached for the
|
||||
* day, 15-minute data is cached for 15 minutes, etc. Historical data (end date
|
||||
* in the past) is cached indefinitely.
|
||||
* <p>
|
||||
* <strong>Unit Testing:</strong> For unit testing with a mock HttpClient, use
|
||||
* the constructor:
|
||||
*
|
||||
* <pre>
|
||||
* HttpClientWrapper mockHttpClient = mock(HttpClientWrapper.class);
|
||||
* CoinbaseHttpBarSeriesDataSource loader = new CoinbaseHttpBarSeriesDataSource(mockHttpClient);
|
||||
* // Use loader instance methods or inject into your code
|
||||
* </pre>
|
||||
* <p>
|
||||
* <strong>API Limits:</strong> Coinbase API has a maximum of 350 candles per
|
||||
* request. This implementation automatically paginates large requests into
|
||||
* multiple API calls and merges the results.
|
||||
* <p>
|
||||
* <strong>Note:</strong> This uses Coinbase's public market data endpoint which
|
||||
* does not require authentication. For production use with higher rate limits,
|
||||
* consider using authenticated endpoints.
|
||||
*
|
||||
* @since 0.20
|
||||
*/
|
||||
public class CoinbaseHttpBarSeriesDataSource extends AbstractHttpBarSeriesDataSource {
|
||||
|
||||
public static final String COINBASE_API_URL = "https://api.coinbase.com/api/v3/brokerage/market/products/";
|
||||
public static final int MAX_CANDLES_PER_REQUEST = 350;
|
||||
|
||||
private static final Logger LOG = LogManager.getLogger(CoinbaseHttpBarSeriesDataSource.class);
|
||||
|
||||
@Override
|
||||
public String getSourceName() {
|
||||
return "Coinbase";
|
||||
}
|
||||
|
||||
private static final HttpClientWrapper DEFAULT_HTTP_CLIENT = new DefaultHttpClientWrapper();
|
||||
private static final CoinbaseHttpBarSeriesDataSource DEFAULT_INSTANCE = new CoinbaseHttpBarSeriesDataSource(
|
||||
DEFAULT_HTTP_CLIENT);
|
||||
|
||||
/**
|
||||
* Creates a new CoinbaseHttpBarSeriesDataSource with a default HttpClient. For
|
||||
* unit testing, use {@link #CoinbaseHttpBarSeriesDataSource(HttpClientWrapper)}
|
||||
* to inject a mock HttpClientWrapper.
|
||||
*/
|
||||
public CoinbaseHttpBarSeriesDataSource() {
|
||||
super(DEFAULT_HTTP_CLIENT, false);
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a new CoinbaseHttpBarSeriesDataSource with a default HttpClient and
|
||||
* caching option.
|
||||
*
|
||||
* @param enableResponseCaching if true, responses will be cached to disk for
|
||||
* faster subsequent requests
|
||||
*/
|
||||
public CoinbaseHttpBarSeriesDataSource(boolean enableResponseCaching) {
|
||||
super(DEFAULT_HTTP_CLIENT, enableResponseCaching);
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a new CoinbaseHttpBarSeriesDataSource with a default HttpClient and
|
||||
* custom cache directory. Response caching is automatically enabled when a
|
||||
* cache directory is specified.
|
||||
*
|
||||
* @param responseCacheDir the directory path for caching responses (can be
|
||||
* relative or absolute)
|
||||
*/
|
||||
public CoinbaseHttpBarSeriesDataSource(String responseCacheDir) {
|
||||
super(DEFAULT_HTTP_CLIENT, responseCacheDir);
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a new CoinbaseHttpBarSeriesDataSource with the specified
|
||||
* HttpClientWrapper. This constructor allows dependency injection of a mock
|
||||
* HttpClientWrapper for unit testing.
|
||||
*
|
||||
* @param httpClient the HttpClientWrapper to use for API requests (can be a
|
||||
* mock for testing)
|
||||
*/
|
||||
public CoinbaseHttpBarSeriesDataSource(HttpClientWrapper httpClient) {
|
||||
super(httpClient, false);
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a new CoinbaseHttpBarSeriesDataSource with the specified
|
||||
* HttpClientWrapper and caching option. This constructor allows dependency
|
||||
* injection of a mock HttpClientWrapper for unit testing and enables response
|
||||
* caching.
|
||||
*
|
||||
* @param httpClient the HttpClientWrapper to use for API requests
|
||||
* (can be a mock for testing)
|
||||
* @param enableResponseCaching if true, responses will be cached to disk for
|
||||
* faster subsequent requests
|
||||
*/
|
||||
public CoinbaseHttpBarSeriesDataSource(HttpClientWrapper httpClient, boolean enableResponseCaching) {
|
||||
super(httpClient, enableResponseCaching);
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a new CoinbaseHttpBarSeriesDataSource with the specified HttpClient.
|
||||
* This is a convenience constructor that wraps the HttpClient in a
|
||||
* DefaultHttpClientWrapper.
|
||||
*
|
||||
* @param httpClient the HttpClient to use for API requests
|
||||
*/
|
||||
public CoinbaseHttpBarSeriesDataSource(HttpClient httpClient) {
|
||||
super(httpClient, false);
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a new CoinbaseHttpBarSeriesDataSource with the specified HttpClient
|
||||
* and caching option.
|
||||
*
|
||||
* @param httpClient the HttpClient to use for API requests
|
||||
* @param enableResponseCaching if true, responses will be cached to disk for
|
||||
* faster subsequent requests
|
||||
*/
|
||||
public CoinbaseHttpBarSeriesDataSource(HttpClient httpClient, boolean enableResponseCaching) {
|
||||
super(httpClient, enableResponseCaching);
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a new CoinbaseHttpBarSeriesDataSource with the specified
|
||||
* HttpClientWrapper and custom cache directory. Response caching is
|
||||
* automatically enabled when a cache directory is specified.
|
||||
*
|
||||
* @param httpClient the HttpClientWrapper to use for API requests (can be
|
||||
* a mock for testing)
|
||||
* @param responseCacheDir the directory path for caching responses (can be
|
||||
* relative or absolute)
|
||||
*/
|
||||
public CoinbaseHttpBarSeriesDataSource(HttpClientWrapper httpClient, String responseCacheDir) {
|
||||
super(httpClient, responseCacheDir);
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a new CoinbaseHttpBarSeriesDataSource with the specified HttpClient
|
||||
* and custom cache directory. Response caching is automatically enabled when a
|
||||
* cache directory is specified.
|
||||
*
|
||||
* @param httpClient the HttpClient to use for API requests
|
||||
* @param responseCacheDir the directory path for caching responses (can be
|
||||
* relative or absolute)
|
||||
*/
|
||||
public CoinbaseHttpBarSeriesDataSource(HttpClient httpClient, String responseCacheDir) {
|
||||
super(httpClient, responseCacheDir);
|
||||
}
|
||||
|
||||
/**
|
||||
* Loads historical OHLCV data for a given product ID within a specified date
|
||||
* range. This is the base method that all other convenience methods delegate
|
||||
* to.
|
||||
* <p>
|
||||
* <strong>Automatic Pagination:</strong> If the requested date range would
|
||||
* exceed 350 candles (Coinbase's maximum per request), this method
|
||||
* automatically splits the request into multiple smaller chunks, fetches them
|
||||
* sequentially, and merges the results into a single BarSeries. This ensures
|
||||
* reliable data retrieval for large date ranges while respecting API limits.
|
||||
*
|
||||
* @param productId the product ID (e.g., "BTC-USD", "ETH-USD")
|
||||
* @param interval the bar interval (must be one of the supported Coinbase
|
||||
* intervals)
|
||||
* @param startDateTime the start date/time for the data range (inclusive)
|
||||
* @param endDateTime the end date/time for the data range (inclusive)
|
||||
* @return a BarSeries containing the historical data, or null if the request
|
||||
* fails
|
||||
*/
|
||||
public static BarSeries loadSeries(String productId, CoinbaseInterval interval, Instant startDateTime,
|
||||
Instant endDateTime) {
|
||||
return DEFAULT_INSTANCE.loadSeriesInstance(productId, interval, startDateTime, endDateTime);
|
||||
}
|
||||
|
||||
/**
|
||||
* Loads historical OHLCV data for a given product ID with a specified number of
|
||||
* bars. The end date/time is set to the current time, and the start date/time
|
||||
* is calculated based on the bar count and interval.
|
||||
* <p>
|
||||
* <strong>Note:</strong> If the calculated date range would exceed 350 candles,
|
||||
* this method will automatically paginate the request into multiple API calls
|
||||
* and merge the results. This ensures reliable data retrieval for large bar
|
||||
* counts.
|
||||
*
|
||||
* @param productId the product ID (e.g., "BTC-USD", "ETH-USD")
|
||||
* @param interval the bar interval (must be one of the supported Coinbase
|
||||
* intervals)
|
||||
* @param barCount the number of bars to fetch
|
||||
* @return a BarSeries containing the historical data, or null if the request
|
||||
* fails
|
||||
*/
|
||||
public static BarSeries loadSeries(String productId, CoinbaseInterval interval, int barCount) {
|
||||
if (barCount <= 0) {
|
||||
LOG.error("Bar count must be greater than 0");
|
||||
return null;
|
||||
}
|
||||
|
||||
Instant endDateTime = Instant.now();
|
||||
Duration totalDuration = interval.getDuration().multipliedBy(barCount);
|
||||
Instant startDateTime = endDateTime.minus(totalDuration);
|
||||
|
||||
return loadSeries(productId, interval, startDateTime, endDateTime);
|
||||
}
|
||||
|
||||
/**
|
||||
* Loads historical OHLCV data for a given product ID with daily bars.
|
||||
* Convenience method that uses the number of days to calculate the date range.
|
||||
*
|
||||
* @param productId the product ID (e.g., "BTC-USD", "ETH-USD")
|
||||
* @param days the number of days of historical data to fetch
|
||||
* @return a BarSeries containing the historical data, or null if the request
|
||||
* fails
|
||||
*/
|
||||
public static BarSeries loadSeries(String productId, int days) {
|
||||
return loadSeries(productId, CoinbaseInterval.ONE_DAY, days);
|
||||
}
|
||||
|
||||
/**
|
||||
* Loads historical OHLCV data for a given product ID with a specified interval.
|
||||
* Convenience method that uses the number of days to calculate the date range.
|
||||
*
|
||||
* @param productId the product ID (e.g., "BTC-USD", "ETH-USD")
|
||||
* @param days the number of days of historical data to fetch
|
||||
* @param interval the bar interval (must be one of the supported Coinbase
|
||||
* intervals)
|
||||
* @return a BarSeries containing the historical data, or null if the request
|
||||
* fails
|
||||
*/
|
||||
public static BarSeries loadSeries(String productId, int days, CoinbaseInterval interval) {
|
||||
if (days <= 0) {
|
||||
LOG.error("Days must be greater than 0");
|
||||
return null;
|
||||
}
|
||||
|
||||
Instant endDateTime = Instant.now();
|
||||
Instant startDateTime = endDateTime.minusSeconds(days * 86400L);
|
||||
|
||||
return loadSeries(productId, interval, startDateTime, endDateTime);
|
||||
}
|
||||
|
||||
/**
|
||||
* Parses the Coinbase API JSON response into a BarSeries using
|
||||
* AdaptiveBarSeriesTypeAdapter.
|
||||
* <p>
|
||||
* This method reuses the existing AdaptiveBarSeriesTypeAdapter which already
|
||||
* supports Coinbase format and handles null values. The adapter correctly
|
||||
* interprets Coinbase's "start" field as the start of the candle period and
|
||||
* calculates the end time as start + interval. The known interval from the API
|
||||
* request is used directly.
|
||||
*
|
||||
* @param jsonResponse the JSON response string from Coinbase API
|
||||
* @param productId the product ID (used as series name)
|
||||
* @param barInterval the known bar interval from the API request
|
||||
* @return a BarSeries containing the parsed data, or null if parsing fails
|
||||
*/
|
||||
private static BarSeries parseCoinbaseResponse(String jsonResponse, String productId, Duration barInterval) {
|
||||
try {
|
||||
JsonObject root = JsonParser.parseString(jsonResponse).getAsJsonObject();
|
||||
|
||||
// Use AdaptiveBarSeriesTypeAdapter's static helper method which handles
|
||||
// Coinbase format correctly (treats "start" as start time, calculates end as
|
||||
// start + interval)
|
||||
// and uses the known interval from the API request
|
||||
BarSeries series = AdaptiveBarSeriesTypeAdapter.parseCoinbaseFormat(root, productId, barInterval);
|
||||
|
||||
if (series == null || series.isEmpty()) {
|
||||
LOG.error("No candles found in Coinbase response for product: {}", productId);
|
||||
return null;
|
||||
}
|
||||
|
||||
LOG.debug("Successfully loaded {} bars for product {}", series.getBarCount(), productId);
|
||||
return series;
|
||||
|
||||
} catch (Exception e) {
|
||||
LOG.error("Error parsing Coinbase response for product {}: {}", productId, e.getMessage(), e);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Merges multiple BarSeries into a single BarSeries, removing duplicates and
|
||||
* sorting chronologically. Uses a TreeMap keyed by timestamp to automatically
|
||||
* handle deduplication and sorting.
|
||||
*
|
||||
* @param chunks list of BarSeries to merge
|
||||
* @param productId the product ID (for the merged series name)
|
||||
* @param barInterval the bar interval
|
||||
* @return a merged BarSeries
|
||||
*/
|
||||
private static BarSeries mergeBarSeries(List<BarSeries> chunks, String productId, Duration barInterval) {
|
||||
// Use TreeMap to automatically sort by timestamp and deduplicate
|
||||
TreeMap<Instant, BarData> barMap = new TreeMap<>();
|
||||
|
||||
// Collect all bars from all chunks
|
||||
for (BarSeries chunk : chunks) {
|
||||
for (int i = 0; i < chunk.getBarCount(); i++) {
|
||||
var bar = chunk.getBar(i);
|
||||
Instant endTime = bar.getEndTime();
|
||||
|
||||
// If we already have a bar at this timestamp, keep the first one
|
||||
barMap.putIfAbsent(endTime, new BarData(bar));
|
||||
}
|
||||
}
|
||||
|
||||
// Build the merged series
|
||||
BarSeries merged = new BaseBarSeriesBuilder().withName(productId).build();
|
||||
|
||||
for (BarData barData : barMap.values()) {
|
||||
merged.barBuilder()
|
||||
.timePeriod(barInterval)
|
||||
.endTime(barData.endTime)
|
||||
.openPrice(barData.open)
|
||||
.highPrice(barData.high)
|
||||
.lowPrice(barData.low)
|
||||
.closePrice(barData.close)
|
||||
.volume(barData.volume)
|
||||
.amount(0)
|
||||
.add();
|
||||
}
|
||||
|
||||
LOG.debug("Merged {} chunks into {} unique bars for product {}", chunks.size(), merged.getBarCount(),
|
||||
productId);
|
||||
return merged;
|
||||
}
|
||||
|
||||
/**
|
||||
* Instance method that loads historical OHLCV data for a given product ID with
|
||||
* a specified number of bars. The end date/time is set to the current time, and
|
||||
* the start date/time is calculated based on the bar count and interval.
|
||||
*
|
||||
* @param productId the product ID (e.g., "BTC-USD", "ETH-USD")
|
||||
* @param interval the bar interval (must be one of the supported Coinbase
|
||||
* intervals)
|
||||
* @param barCount the number of bars to fetch
|
||||
* @return a BarSeries containing the historical data, or null if the request
|
||||
* fails
|
||||
*/
|
||||
public BarSeries loadSeriesInstance(String productId, CoinbaseInterval interval, int barCount) {
|
||||
return loadSeriesInstance(productId, interval, barCount, null);
|
||||
}
|
||||
|
||||
/**
|
||||
* Instance method that loads historical OHLCV data for a given product ID with
|
||||
* a specified number of bars and optional notes for cache file naming. The end
|
||||
* date/time is set to the current time, and the start date/time is calculated
|
||||
* based on the bar count and interval.
|
||||
*
|
||||
* @param productId the product ID (e.g., "BTC-USD", "ETH-USD")
|
||||
* @param interval the bar interval (must be one of the supported Coinbase
|
||||
* intervals)
|
||||
* @param barCount the number of bars to fetch
|
||||
* @param notes optional notes to include in cache filename (for uniqueness,
|
||||
* e.g., test identifiers)
|
||||
* @return a BarSeries containing the historical data, or null if the request
|
||||
* fails
|
||||
*/
|
||||
public BarSeries loadSeriesInstance(String productId, CoinbaseInterval interval, int barCount, String notes) {
|
||||
if (barCount <= 0) {
|
||||
LOG.error("Bar count must be greater than 0");
|
||||
return null;
|
||||
}
|
||||
|
||||
Instant endDateTime = Instant.now();
|
||||
Duration totalDuration = interval.getDuration().multipliedBy(barCount);
|
||||
Instant startDateTime = endDateTime.minus(totalDuration);
|
||||
|
||||
return loadSeriesInstance(productId, interval, startDateTime, endDateTime, notes);
|
||||
}
|
||||
|
||||
@Override
|
||||
public BarSeries loadSeries(String productId, Duration interval, Instant start, Instant end) {
|
||||
if (productId == null || productId.trim().isEmpty()) {
|
||||
throw new IllegalArgumentException("Product ID cannot be null or empty");
|
||||
}
|
||||
if (interval == null || interval.isNegative() || interval.isZero()) {
|
||||
throw new IllegalArgumentException("Interval must be positive");
|
||||
}
|
||||
if (start == null || end == null) {
|
||||
throw new IllegalArgumentException("Start and end dates cannot be null");
|
||||
}
|
||||
if (start.isAfter(end)) {
|
||||
throw new IllegalArgumentException("Start date must be before or equal to end date");
|
||||
}
|
||||
|
||||
// Map Duration to CoinbaseInterval
|
||||
CoinbaseInterval cbInterval = mapDurationToInterval(interval);
|
||||
if (cbInterval == null) {
|
||||
LOG.warn("Unsupported interval duration: {}. Falling back to ONE_DAY", interval);
|
||||
cbInterval = CoinbaseInterval.ONE_DAY;
|
||||
}
|
||||
|
||||
return loadSeriesInstance(productId, cbInterval, start, end);
|
||||
}
|
||||
|
||||
@Override
|
||||
public BarSeries loadSeries(String source) {
|
||||
if (source == null || source.trim().isEmpty()) {
|
||||
throw new IllegalArgumentException("Source cannot be null or empty");
|
||||
}
|
||||
|
||||
// Check if it's a cache file path
|
||||
String sourcePrefix = getSourceName().isEmpty() ? "" : getSourceName() + "-";
|
||||
if (source.startsWith(responseCacheDir) || (!sourcePrefix.isEmpty() && source.contains(sourcePrefix))) {
|
||||
Path cacheFile = Paths.get(source);
|
||||
if (Files.exists(cacheFile)) {
|
||||
String cachedResponse = readFromCache(cacheFile);
|
||||
if (cachedResponse != null) {
|
||||
// Try to extract product ID from filename
|
||||
String filename = cacheFile.getFileName().toString();
|
||||
// Format: {sourceName}-PRODUCTID-INTERVAL-START-END[_NOTES].json
|
||||
// Remove extension
|
||||
String baseName = filename.replace(".json", "");
|
||||
String[] parts = baseName.split("-");
|
||||
if (parts.length >= 5) {
|
||||
String productId = parts[1];
|
||||
// Try to determine interval from filename
|
||||
CoinbaseInterval interval = CoinbaseInterval.ONE_DAY; // Default
|
||||
try {
|
||||
interval = parseIntervalFromApiValue(parts[2]);
|
||||
} catch (IllegalArgumentException e) {
|
||||
LOG.debug("Could not parse interval from filename, using default: {}", e.getMessage());
|
||||
}
|
||||
|
||||
return parseCoinbaseResponse(cachedResponse, productId, interval.getDuration());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// If not a cache file, return null (could be extended to parse other formats)
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Maps a Duration to the closest matching CoinbaseInterval.
|
||||
*
|
||||
* @param duration the duration to map
|
||||
* @return the matching CoinbaseInterval, or null if no close match is found
|
||||
*/
|
||||
private CoinbaseInterval mapDurationToInterval(Duration duration) {
|
||||
long seconds = duration.getSeconds();
|
||||
for (CoinbaseInterval interval : CoinbaseInterval.values()) {
|
||||
if (interval.getDuration().getSeconds() == seconds) {
|
||||
return interval;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parses a CoinbaseInterval from its API value string.
|
||||
*
|
||||
* @param apiValue the API value (e.g., "ONE_MINUTE", "ONE_DAY")
|
||||
* @return the matching CoinbaseInterval
|
||||
* @throws IllegalArgumentException if no matching interval is found
|
||||
*/
|
||||
private CoinbaseInterval parseIntervalFromApiValue(String apiValue) {
|
||||
for (CoinbaseInterval interval : CoinbaseInterval.values()) {
|
||||
if (interval.getApiValue().equals(apiValue)) {
|
||||
return interval;
|
||||
}
|
||||
}
|
||||
throw new IllegalArgumentException("Unknown interval API value: " + apiValue);
|
||||
}
|
||||
|
||||
/**
|
||||
* Instance method that performs the actual loading logic. This method uses the
|
||||
* instance's HttpClient (which can be injected for testing).
|
||||
*/
|
||||
public BarSeries loadSeriesInstance(String productId, CoinbaseInterval interval, Instant startDateTime,
|
||||
Instant endDateTime) {
|
||||
return loadSeriesInstance(productId, interval, startDateTime, endDateTime, null);
|
||||
}
|
||||
|
||||
/**
|
||||
* Instance method that performs the actual loading logic with optional notes.
|
||||
* This method uses the instance's HttpClient (which can be injected for
|
||||
* testing).
|
||||
*
|
||||
* @param productId the product ID
|
||||
* @param interval the interval
|
||||
* @param startDateTime the start date/time
|
||||
* @param endDateTime the end date/time
|
||||
* @param notes optional notes to include in cache filename (for
|
||||
* uniqueness)
|
||||
* @return the BarSeries or null if request fails
|
||||
*/
|
||||
public BarSeries loadSeriesInstance(String productId, CoinbaseInterval interval, Instant startDateTime,
|
||||
Instant endDateTime, String notes) {
|
||||
if (productId == null || productId.trim().isEmpty()) {
|
||||
LOG.error("Product ID cannot be null or empty");
|
||||
return null;
|
||||
}
|
||||
|
||||
if (startDateTime == null || endDateTime == null) {
|
||||
LOG.error("Start and end date/time cannot be null");
|
||||
return null;
|
||||
}
|
||||
|
||||
if (startDateTime.isAfter(endDateTime)) {
|
||||
LOG.error("Start date/time must be before or equal to end date/time");
|
||||
return null;
|
||||
}
|
||||
|
||||
// Calculate if we need pagination (max 350 candles per request)
|
||||
Duration requestedRange = Duration.between(startDateTime, endDateTime);
|
||||
long requestedBars = requestedRange.dividedBy(interval.getDuration());
|
||||
|
||||
if (requestedBars > MAX_CANDLES_PER_REQUEST) {
|
||||
LOG.debug(
|
||||
"Requested date range would result in {} bars (max {}). "
|
||||
+ "Splitting into multiple requests and combining results.",
|
||||
requestedBars, MAX_CANDLES_PER_REQUEST);
|
||||
return loadSeriesPaginated(productId, interval, startDateTime, endDateTime, notes);
|
||||
}
|
||||
|
||||
// Single request for smaller ranges
|
||||
return loadSeriesSingleRequest(productId, interval, startDateTime, endDateTime, notes);
|
||||
}
|
||||
|
||||
/**
|
||||
* Generates the cache file path for a given request.
|
||||
*
|
||||
* @param productId the product ID
|
||||
* @param interval the interval
|
||||
* @param startDateTime the start date/time (will be truncated)
|
||||
* @param endDateTime the end date/time (will be truncated)
|
||||
* @param notes optional notes section to append to filename (can be
|
||||
* null or empty)
|
||||
* @return the cache file path
|
||||
*/
|
||||
private Path getCacheFilePath(String productId, CoinbaseInterval interval, Instant startDateTime,
|
||||
Instant endDateTime, String notes) {
|
||||
return getCacheFilePath(productId, startDateTime, endDateTime, interval.getDuration(), notes);
|
||||
}
|
||||
|
||||
/**
|
||||
* Generates the cache file path for a given request (without notes).
|
||||
*
|
||||
* @param productId the product ID
|
||||
* @param interval the interval
|
||||
* @param startDateTime the start date/time (will be truncated)
|
||||
* @param endDateTime the end date/time (will be truncated)
|
||||
* @return the cache file path
|
||||
*/
|
||||
private Path getCacheFilePath(String productId, CoinbaseInterval interval, Instant startDateTime,
|
||||
Instant endDateTime) {
|
||||
return getCacheFilePath(productId, interval, startDateTime, endDateTime, null);
|
||||
}
|
||||
|
||||
/**
|
||||
* Makes a single API request for the specified date range with optional notes.
|
||||
* This is used for requests that don't exceed 350 candles. If caching is
|
||||
* enabled, checks cache first before making the API request.
|
||||
*
|
||||
* @param productId the product ID
|
||||
* @param interval the interval
|
||||
* @param startDateTime the start date/time
|
||||
* @param endDateTime the end date/time
|
||||
* @param notes optional notes to include in cache filename (for
|
||||
* uniqueness)
|
||||
* @return the BarSeries or null if request fails
|
||||
*/
|
||||
private BarSeries loadSeriesSingleRequest(String productId, CoinbaseInterval interval, Instant startDateTime,
|
||||
Instant endDateTime, String notes) {
|
||||
// Check cache first if caching is enabled
|
||||
if (enableResponseCaching) {
|
||||
// Try exact match first (with or without notes)
|
||||
Path cacheFile = getCacheFilePath(productId, interval, startDateTime, endDateTime, notes);
|
||||
if (isCacheValid(cacheFile, interval.getDuration(), endDateTime)) {
|
||||
String cachedResponse = readFromCache(cacheFile);
|
||||
if (cachedResponse != null) {
|
||||
LOG.debug("Using cached response for {} ({} to {})", productId, startDateTime, endDateTime);
|
||||
return parseCoinbaseResponse(cachedResponse, productId, interval.getDuration());
|
||||
}
|
||||
}
|
||||
// Also try without notes (for backward compatibility)
|
||||
if (notes != null && !notes.trim().isEmpty()) {
|
||||
Path cacheFileNoNotes = getCacheFilePath(productId, interval, startDateTime, endDateTime);
|
||||
if (isCacheValid(cacheFileNoNotes, interval.getDuration(), endDateTime)) {
|
||||
String cachedResponse = readFromCache(cacheFileNoNotes);
|
||||
if (cachedResponse != null) {
|
||||
LOG.debug("Using cached response for {} ({} to {})", productId, startDateTime, endDateTime);
|
||||
return parseCoinbaseResponse(cachedResponse, productId, interval.getDuration());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
String encodedProductId = URLEncoder.encode(productId.trim(), StandardCharsets.UTF_8);
|
||||
long startTimestamp = startDateTime.getEpochSecond();
|
||||
long endTimestamp = endDateTime.getEpochSecond();
|
||||
|
||||
String url = String.format("%s%s/candles?start=%d&end=%d&granularity=%s&limit=%d", COINBASE_API_URL,
|
||||
encodedProductId, startTimestamp, endTimestamp, interval.getApiValue(), MAX_CANDLES_PER_REQUEST);
|
||||
|
||||
LOG.trace("Fetching data from Coinbase: {}", url);
|
||||
|
||||
HttpRequest request = HttpRequest.newBuilder()
|
||||
.uri(URI.create(url))
|
||||
.header("Accept", "application/json")
|
||||
.timeout(Duration.ofSeconds(30))
|
||||
.GET()
|
||||
.build();
|
||||
|
||||
HttpResponseWrapper<String> response = httpClient.send(request, HttpResponse.BodyHandlers.ofString());
|
||||
|
||||
if (response.statusCode() != 200) {
|
||||
LOG.error("Coinbase API returned status code: {}", response.statusCode());
|
||||
return null;
|
||||
}
|
||||
|
||||
String responseBody = response.body();
|
||||
LOG.trace("Response body: {}", responseBody);
|
||||
|
||||
// Cache the response if caching is enabled
|
||||
if (enableResponseCaching) {
|
||||
Path cacheFile = getCacheFilePath(productId, interval, startDateTime, endDateTime, notes);
|
||||
writeToCache(cacheFile, responseBody);
|
||||
}
|
||||
|
||||
return parseCoinbaseResponse(responseBody, productId, interval.getDuration());
|
||||
|
||||
} catch (IOException | InterruptedException e) {
|
||||
LOG.error("Error fetching data from Coinbase for product {}: {}", productId, e.getMessage(), e);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Loads data by splitting a large date range into multiple smaller requests
|
||||
* (pagination). Each chunk respects the 350 candle limit, and results are
|
||||
* merged chronologically.
|
||||
*
|
||||
* @param productId the product ID
|
||||
* @param interval the bar interval
|
||||
* @param startDateTime the start date/time
|
||||
* @param endDateTime the end date/time
|
||||
* @param notes optional notes to include in cache filename (for
|
||||
* uniqueness)
|
||||
* @return a BarSeries containing all merged data, or null if all requests fail
|
||||
*/
|
||||
private BarSeries loadSeriesPaginated(String productId, CoinbaseInterval interval, Instant startDateTime,
|
||||
Instant endDateTime, String notes) {
|
||||
List<BarSeries> chunks = new ArrayList<>();
|
||||
Instant currentStart = startDateTime;
|
||||
int requestCount = 0;
|
||||
|
||||
// Calculate chunk size (350 candles worth of time)
|
||||
Duration chunkSize = interval.getDuration().multipliedBy(MAX_CANDLES_PER_REQUEST);
|
||||
|
||||
// Calculate number of chunks needed
|
||||
Duration totalRange = Duration.between(startDateTime, endDateTime);
|
||||
int estimatedChunks = (int) Math.ceil((double) totalRange.toSeconds() / chunkSize.toSeconds());
|
||||
LOG.trace("Splitting request into approximately {} chunks", estimatedChunks);
|
||||
|
||||
while (currentStart.isBefore(endDateTime)) {
|
||||
// Calculate chunk end time (don't exceed the requested end time)
|
||||
Instant chunkEnd = currentStart.plus(chunkSize);
|
||||
if (chunkEnd.isAfter(endDateTime)) {
|
||||
chunkEnd = endDateTime;
|
||||
}
|
||||
|
||||
requestCount++;
|
||||
LOG.trace("Fetching chunk {}/? ({} to {})", requestCount, currentStart, chunkEnd);
|
||||
|
||||
BarSeries chunk = loadSeriesSingleRequest(productId, interval, currentStart, chunkEnd, notes);
|
||||
if (chunk != null && chunk.getBarCount() > 0) {
|
||||
chunks.add(chunk);
|
||||
LOG.trace("Successfully loaded chunk {} with {} bars", requestCount, chunk.getBarCount());
|
||||
} else {
|
||||
LOG.warn("Chunk {} returned no data or failed", requestCount);
|
||||
}
|
||||
|
||||
// Move to next chunk (start from the end of current chunk)
|
||||
currentStart = chunkEnd;
|
||||
|
||||
// If we've reached the end, break
|
||||
if (chunkEnd.equals(endDateTime) || !currentStart.isBefore(endDateTime)) {
|
||||
break;
|
||||
}
|
||||
|
||||
// Add a small delay between requests to avoid rate limiting
|
||||
try {
|
||||
Thread.sleep(100); // 100ms delay between requests
|
||||
} catch (InterruptedException e) {
|
||||
Thread.currentThread().interrupt();
|
||||
LOG.warn("Interrupted during pagination delay");
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (chunks.isEmpty()) {
|
||||
LOG.error("All paginated requests failed for product {}", productId);
|
||||
return null;
|
||||
}
|
||||
|
||||
LOG.debug("Successfully fetched {} chunks, merging {} total bars", chunks.size(),
|
||||
chunks.stream().mapToInt(BarSeries::getBarCount).sum());
|
||||
|
||||
return mergeBarSeries(chunks, productId, interval.getDuration());
|
||||
}
|
||||
|
||||
/**
|
||||
* Supported intervals for Coinbase API. These correspond to the intervals that
|
||||
* Coinbase's Advanced Trade API supports.
|
||||
*/
|
||||
public enum CoinbaseInterval {
|
||||
/**
|
||||
* 1 minute bars
|
||||
*/
|
||||
ONE_MINUTE(Duration.ofMinutes(1), "ONE_MINUTE"),
|
||||
/**
|
||||
* 5 minute bars
|
||||
*/
|
||||
FIVE_MINUTE(Duration.ofMinutes(5), "FIVE_MINUTE"),
|
||||
/**
|
||||
* 15 minute bars
|
||||
*/
|
||||
FIFTEEN_MINUTE(Duration.ofMinutes(15), "FIFTEEN_MINUTE"),
|
||||
/**
|
||||
* 30 minute bars
|
||||
*/
|
||||
THIRTY_MINUTE(Duration.ofMinutes(30), "THIRTY_MINUTE"),
|
||||
/**
|
||||
* 1 hour bars
|
||||
*/
|
||||
ONE_HOUR(Duration.ofHours(1), "ONE_HOUR"),
|
||||
/**
|
||||
* 2 hour bars
|
||||
*/
|
||||
TWO_HOUR(Duration.ofHours(2), "TWO_HOUR"),
|
||||
/**
|
||||
* 4 hour bars
|
||||
*/
|
||||
FOUR_HOUR(Duration.ofHours(4), "FOUR_HOUR"),
|
||||
/**
|
||||
* 6 hour bars
|
||||
*/
|
||||
SIX_HOUR(Duration.ofHours(6), "SIX_HOUR"),
|
||||
/**
|
||||
* 1 day bars
|
||||
*/
|
||||
ONE_DAY(Duration.ofDays(1), "ONE_DAY");
|
||||
|
||||
private final Duration duration;
|
||||
private final String apiValue;
|
||||
|
||||
CoinbaseInterval(Duration duration, String apiValue) {
|
||||
this.duration = duration;
|
||||
this.apiValue = apiValue;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the Duration for this interval.
|
||||
*
|
||||
* @return the Duration
|
||||
*/
|
||||
public Duration getDuration() {
|
||||
return duration;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the API string value for this interval.
|
||||
*
|
||||
* @return the API string value
|
||||
*/
|
||||
public String getApiValue() {
|
||||
return apiValue;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Helper class to hold bar data during merging.
|
||||
*/
|
||||
private static class BarData {
|
||||
final Instant endTime;
|
||||
final double open;
|
||||
final double high;
|
||||
final double low;
|
||||
final double close;
|
||||
final double volume;
|
||||
|
||||
BarData(Bar bar) {
|
||||
this.endTime = bar.getEndTime();
|
||||
this.open = bar.getOpenPrice().doubleValue();
|
||||
this.high = bar.getHighPrice().doubleValue();
|
||||
this.low = bar.getLowPrice().doubleValue();
|
||||
this.close = bar.getClosePrice().doubleValue();
|
||||
this.volume = bar.getVolume().doubleValue();
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
+250
@@ -0,0 +1,250 @@
|
||||
/*
|
||||
* SPDX-License-Identifier: MIT
|
||||
*/
|
||||
package ta4jexamples.datasources;
|
||||
|
||||
import com.opencsv.CSVParserBuilder;
|
||||
import com.opencsv.CSVReader;
|
||||
import com.opencsv.CSVReaderBuilder;
|
||||
import com.opencsv.exceptions.CsvValidationException;
|
||||
import org.apache.logging.log4j.LogManager;
|
||||
import org.apache.logging.log4j.Logger;
|
||||
import org.ta4j.core.BarSeries;
|
||||
import org.ta4j.core.BaseBarSeriesBuilder;
|
||||
import ta4jexamples.datasources.file.AbstractFileBarSeriesDataSource;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.InputStreamReader;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.time.Duration;
|
||||
import java.time.Instant;
|
||||
import java.time.LocalDate;
|
||||
import java.time.ZoneOffset;
|
||||
import java.time.format.DateTimeFormatter;
|
||||
|
||||
/**
|
||||
* This class builds a Ta4j bar series from a CSV file containing bars.
|
||||
* <p>
|
||||
* Implements {@link BarSeriesDataSource} to support domain-driven loading by
|
||||
* ticker, interval, and date range. Searches for CSV files matching the
|
||||
* specified criteria in the classpath.
|
||||
*/
|
||||
public class CsvFileBarSeriesDataSource extends AbstractFileBarSeriesDataSource {
|
||||
|
||||
private static final Logger LOG = LogManager.getLogger(CsvFileBarSeriesDataSource.class);
|
||||
private static final DateTimeFormatter DATE_FORMAT = DateTimeFormatter.ofPattern("yyyy-MM-dd");
|
||||
private static final String DEFAULT_APPLE_BAR_FILE = "AAPL-PT1D-20130102_20131231.csv";
|
||||
|
||||
/**
|
||||
* Creates a new CsvFileBarSeriesDataSource with no source prefix.
|
||||
*/
|
||||
public CsvFileBarSeriesDataSource() {
|
||||
super("");
|
||||
}
|
||||
|
||||
@Override
|
||||
protected String getFileExtension() {
|
||||
return "csv";
|
||||
}
|
||||
|
||||
@Override
|
||||
protected BarSeries searchAndLoadFile(String ticker, String intervalStr, String sourcePrefix,
|
||||
String startDateTimeStr, String endDateTimeStr, String startDateStr, String endDateStr, Duration interval,
|
||||
Instant start, Instant end) {
|
||||
// Try exact pattern with interval-appropriate format:
|
||||
// {ticker}-{interval}-{startDateTime}_{endDateTime}.csv
|
||||
String exactPattern = ticker.toUpperCase() + "-" + intervalStr + "-" + startDateTimeStr + "_" + endDateTimeStr
|
||||
+ ".csv";
|
||||
BarSeries series = loadCsvSeries(exactPattern);
|
||||
if (series != null && !series.isEmpty()) {
|
||||
return series;
|
||||
}
|
||||
|
||||
// Fallback to date-only format for existing files
|
||||
String exactPatternDateOnly = ticker.toUpperCase() + "-" + intervalStr + "-" + startDateStr + "_" + endDateStr
|
||||
+ ".csv";
|
||||
series = loadCsvSeries(exactPatternDateOnly);
|
||||
if (series != null && !series.isEmpty()) {
|
||||
return series;
|
||||
}
|
||||
|
||||
// Try broader pattern: {ticker}-*-{startDateTime}_*.csv
|
||||
String broaderPattern = ticker.toUpperCase() + "-*-" + startDateTimeStr + "_*.csv";
|
||||
series = searchAndLoadCsvFile(broaderPattern, start, end);
|
||||
if (series != null && !series.isEmpty()) {
|
||||
return series;
|
||||
}
|
||||
|
||||
// Fallback to date-only format for broader pattern
|
||||
String broaderPatternDateOnly = ticker.toUpperCase() + "-*-" + startDateStr + "_*.csv";
|
||||
series = searchAndLoadCsvFile(broaderPatternDateOnly, start, end);
|
||||
if (series != null && !series.isEmpty()) {
|
||||
return series;
|
||||
}
|
||||
|
||||
// Try even broader: {ticker}-*.csv (then filter by date range)
|
||||
String broadestPattern = ticker.toUpperCase() + "-*.csv";
|
||||
series = searchAndLoadCsvFile(broadestPattern, start, end);
|
||||
if (series != null && !series.isEmpty()) {
|
||||
return filterSeriesByDateRange(series, start, end);
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public BarSeries loadSeries(String source) {
|
||||
if (source == null || source.trim().isEmpty()) {
|
||||
throw new IllegalArgumentException("Source cannot be null or empty");
|
||||
}
|
||||
return loadCsvSeries(source);
|
||||
}
|
||||
|
||||
/**
|
||||
* Searches for a CSV file matching the pattern and loads it if found.
|
||||
*
|
||||
* @param pattern the filename pattern to search for (supports wildcards)
|
||||
* @param start the start date (for validation)
|
||||
* @param end the end date (for validation)
|
||||
* @return the loaded BarSeries, or null if not found
|
||||
*/
|
||||
private BarSeries searchAndLoadCsvFile(String pattern, Instant start, Instant end) {
|
||||
// Try direct pattern match as resource
|
||||
if (!pattern.contains("*")) {
|
||||
return loadCsvSeries(pattern);
|
||||
}
|
||||
|
||||
// Count wildcards to determine replacement strategy
|
||||
long wildcardCount = pattern.chars().filter(ch -> ch == '*').count();
|
||||
|
||||
if (wildcardCount == 1) {
|
||||
// Single wildcard: try common interval values
|
||||
String[] intervalVariations = { "PT1D", "PT5M", "PT1H", "" };
|
||||
for (String interval : intervalVariations) {
|
||||
String variation = pattern.replaceFirst("\\*", interval);
|
||||
BarSeries series = loadCsvSeries(variation);
|
||||
if (series != null && !series.isEmpty()) {
|
||||
return series;
|
||||
}
|
||||
}
|
||||
} else if (wildcardCount >= 2) {
|
||||
// Multiple wildcards: replace the first (interval) with common values,
|
||||
// and the second (end date) with the actual end date from parameters
|
||||
String[] intervalVariations = { "PT1D", "PT5M", "PT1H" };
|
||||
// Format end date in both date-only and datetime formats
|
||||
String endDateStr = end.atZone(ZoneOffset.UTC).format(FILENAME_DATE_FORMAT);
|
||||
String endDateTimeStr = end.atZone(ZoneOffset.UTC)
|
||||
.format(getDateTimeFormatterForInterval(Duration.ofDays(1)));
|
||||
|
||||
for (String interval : intervalVariations) {
|
||||
// Replace first wildcard with interval, second with end date (date-only format)
|
||||
String variation = pattern.replaceFirst("\\*", interval).replaceFirst("\\*", endDateStr);
|
||||
BarSeries series = loadCsvSeries(variation);
|
||||
if (series != null && !series.isEmpty()) {
|
||||
return series;
|
||||
}
|
||||
// Also try with datetime format for the end date
|
||||
variation = pattern.replaceFirst("\\*", interval).replaceFirst("\\*", endDateTimeStr);
|
||||
series = loadCsvSeries(variation);
|
||||
if (series != null && !series.isEmpty()) {
|
||||
return series;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Loads a bar series from the default CSV file.
|
||||
*
|
||||
* @return the bar series loaded from the default CSV file
|
||||
*/
|
||||
public static BarSeries loadSeriesFromFile() {
|
||||
return loadSeriesFromFile(DEFAULT_APPLE_BAR_FILE);
|
||||
}
|
||||
|
||||
/**
|
||||
* Loads a bar series from the specified CSV file. This is a convenience method
|
||||
* that delegates to {@link #loadCsvSeries(String)}.
|
||||
*
|
||||
* @param csvFile the path to the CSV file containing bar data
|
||||
* @return the bar series loaded from the specified CSV file
|
||||
*/
|
||||
public static BarSeries loadSeriesFromFile(String csvFile) {
|
||||
return loadCsvSeries(csvFile);
|
||||
}
|
||||
|
||||
/**
|
||||
* Loads a bar series from a CSV file with the specified filename. The CSV file
|
||||
* is expected to contain stock market data with the following columns: date,
|
||||
* open price, high price, low price, close price, and volume. The date format
|
||||
* is expected to match the predefined DATE_FORMAT.
|
||||
*
|
||||
* @param filename the name of the CSV file to load
|
||||
* @return the bar series containing stock data loaded from the specified CSV
|
||||
* file, or null if the file is not found or empty
|
||||
*/
|
||||
public static BarSeries loadCsvSeries(String filename) {
|
||||
|
||||
var stream = CsvFileBarSeriesDataSource.class.getClassLoader().getResourceAsStream(filename);
|
||||
|
||||
if (stream == null) {
|
||||
LOG.debug("CSV file not found in classpath: {}", filename);
|
||||
return null;
|
||||
}
|
||||
|
||||
var series = new BaseBarSeriesBuilder().withName(filename).build();
|
||||
|
||||
try (stream) {
|
||||
try (InputStreamReader reader = new InputStreamReader(stream, StandardCharsets.UTF_8)) {
|
||||
try (CSVReader csvReader = new CSVReaderBuilder(reader)
|
||||
.withCSVParser(new CSVParserBuilder().withSeparator(',').build())
|
||||
.withSkipLines(1)
|
||||
.build()) {
|
||||
String[] line;
|
||||
while ((line = csvReader.readNext()) != null) {
|
||||
Instant date = LocalDate.parse(line[0], DATE_FORMAT).atStartOfDay(ZoneOffset.UTC).toInstant();
|
||||
double open = Double.parseDouble(line[1]);
|
||||
double high = Double.parseDouble(line[2]);
|
||||
double low = Double.parseDouble(line[3]);
|
||||
double close = Double.parseDouble(line[4]);
|
||||
double volume = Double.parseDouble(line[5]);
|
||||
|
||||
series.barBuilder()
|
||||
.timePeriod(Duration.ofDays(1))
|
||||
.endTime(date)
|
||||
.openPrice(open)
|
||||
.closePrice(close)
|
||||
.highPrice(high)
|
||||
.lowPrice(low)
|
||||
.volume(volume)
|
||||
.amount(0)
|
||||
.add();
|
||||
}
|
||||
} catch (CsvValidationException e) {
|
||||
LOG.error("Unable to load bars from CSV. File is not valid csv.", e);
|
||||
}
|
||||
}
|
||||
} catch (IOException ioe) {
|
||||
LOG.error("Unable to load bars from CSV", ioe);
|
||||
} catch (NumberFormatException nfe) {
|
||||
LOG.error("Error while parsing value", nfe);
|
||||
return null;
|
||||
}
|
||||
return series.isEmpty() ? null : series;
|
||||
}
|
||||
|
||||
public static void main(String[] args) {
|
||||
BarSeries series = CsvFileBarSeriesDataSource.loadSeriesFromFile();
|
||||
|
||||
LOG.debug("Series: {} ({})", series.getName(), series.getSeriesPeriodDescription());
|
||||
LOG.debug("Number of bars: {}", series.getBarCount());
|
||||
if (series.isEmpty()) {
|
||||
LOG.warn("Series is empty - no bars loaded from CSV file. Skipping first bar details.");
|
||||
} else {
|
||||
LOG.debug("First bar: \n\tVolume: {}\n\tOpen price: {}\n\tClose price: {}", series.getBar(0).getVolume(),
|
||||
series.getBar(0).getOpenPrice(), series.getBar(0).getClosePrice());
|
||||
}
|
||||
}
|
||||
}
|
||||
+177
@@ -0,0 +1,177 @@
|
||||
/*
|
||||
* SPDX-License-Identifier: MIT
|
||||
*/
|
||||
package ta4jexamples.datasources;
|
||||
|
||||
import com.google.gson.Gson;
|
||||
import com.google.gson.GsonBuilder;
|
||||
import com.google.gson.stream.JsonReader;
|
||||
import org.apache.logging.log4j.LogManager;
|
||||
import org.apache.logging.log4j.Logger;
|
||||
import org.ta4j.core.BarSeries;
|
||||
import ta4jexamples.datasources.file.AbstractFileBarSeriesDataSource;
|
||||
import ta4jexamples.datasources.json.AdaptiveBarSeriesTypeAdapter;
|
||||
|
||||
import java.io.FileInputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.time.Duration;
|
||||
import java.time.Instant;
|
||||
|
||||
/**
|
||||
* A data source for BarSeries objects that can adapt to different JSON formats.
|
||||
* This class provides methods to load BarSeries data from JSON format,
|
||||
* specifically supporting multiple exchange formats such as Binance and
|
||||
* Coinbase. It uses Gson with a custom TypeAdapter to handle the
|
||||
* deserialization process. The data source can read from either an InputStream
|
||||
* or a file path.
|
||||
* <p>
|
||||
* Implements {@link BarSeriesDataSource} to support domain-driven loading by
|
||||
* ticker, interval, and date range. Searches for JSON files matching the
|
||||
* specified criteria in the classpath.
|
||||
*
|
||||
* @since 0.19
|
||||
*/
|
||||
public class JsonFileBarSeriesDataSource extends AbstractFileBarSeriesDataSource {
|
||||
private static final Gson TYPEADAPTER_GSON = new GsonBuilder()
|
||||
.registerTypeAdapter(BarSeries.class, new AdaptiveBarSeriesTypeAdapter())
|
||||
.create();
|
||||
private static final Logger LOG = LogManager.getLogger(JsonFileBarSeriesDataSource.class);
|
||||
|
||||
/**
|
||||
* Creates a new JsonFileBarSeriesDataSource with no source prefix.
|
||||
*/
|
||||
public JsonFileBarSeriesDataSource() {
|
||||
super("");
|
||||
}
|
||||
|
||||
/**
|
||||
* Default instance for backward compatibility with static method calls. Use
|
||||
* this instance when migrating from static methods to instance methods.
|
||||
*/
|
||||
public static final JsonFileBarSeriesDataSource DEFAULT_INSTANCE = new JsonFileBarSeriesDataSource();
|
||||
|
||||
@Override
|
||||
protected String getFileExtension() {
|
||||
return "json";
|
||||
}
|
||||
|
||||
@Override
|
||||
protected BarSeries searchAndLoadFile(String ticker, String intervalStr, String sourcePrefix,
|
||||
String startDateTimeStr, String endDateTimeStr, String startDateStr, String endDateStr, Duration interval,
|
||||
Instant start, Instant end) {
|
||||
// Try with exchange prefixes (Coinbase-, Binance-)
|
||||
// NOTE: All branches must call filterSeriesByDateRange() to ensure data is
|
||||
// filtered
|
||||
// to the requested date range, even when files contain broader date ranges.
|
||||
String[] exchangePrefixes = { "Coinbase-", "Binance-" };
|
||||
for (String exchange : exchangePrefixes) {
|
||||
// Try exact pattern with interval-appropriate format:
|
||||
// {Exchange}-{ticker}-{interval}-{startDateTime}_{endDateTime}.json
|
||||
String pattern = exchange + ticker.toUpperCase() + "-" + intervalStr + "-" + startDateTimeStr + "_"
|
||||
+ endDateTimeStr + ".json";
|
||||
BarSeries series = loadFromSource(pattern);
|
||||
if (series != null && !series.isEmpty()) {
|
||||
return filterSeriesByDateRange(series, start, end);
|
||||
}
|
||||
|
||||
// Fallback to date-only format for existing files
|
||||
// NOTE: Date-only format may match files with broader date ranges, so filtering
|
||||
// is required
|
||||
String patternDateOnly = exchange + ticker.toUpperCase() + "-" + intervalStr + "-" + startDateStr + "_"
|
||||
+ endDateStr + ".json";
|
||||
series = loadFromSource(patternDateOnly);
|
||||
if (series != null && !series.isEmpty()) {
|
||||
return filterSeriesByDateRange(series, start, end);
|
||||
}
|
||||
}
|
||||
|
||||
// Try without exchange prefix as fallback (for generic JSON files)
|
||||
String pattern = ticker.toUpperCase() + "-" + intervalStr + "-" + startDateTimeStr + "_" + endDateTimeStr
|
||||
+ ".json";
|
||||
BarSeries series = loadFromSource(pattern);
|
||||
if (series != null && !series.isEmpty()) {
|
||||
return filterSeriesByDateRange(series, start, end);
|
||||
}
|
||||
|
||||
// Fallback to date-only format
|
||||
// NOTE: Date-only format may match files with broader date ranges, so filtering
|
||||
// is required
|
||||
String patternDateOnly = ticker.toUpperCase() + "-" + intervalStr + "-" + startDateStr + "_" + endDateStr
|
||||
+ ".json";
|
||||
series = loadFromSource(patternDateOnly);
|
||||
if (series != null && !series.isEmpty()) {
|
||||
return filterSeriesByDateRange(series, start, end);
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public BarSeries loadSeries(String source) {
|
||||
return loadFromSource(source);
|
||||
}
|
||||
|
||||
@Override
|
||||
public BarSeries loadSeries(InputStream inputStream) {
|
||||
return loadFromStream(inputStream);
|
||||
}
|
||||
|
||||
/**
|
||||
* Internal implementation for loading from a file.
|
||||
*/
|
||||
private BarSeries loadFromSource(String source) {
|
||||
if (source == null || source.trim().isEmpty()) {
|
||||
return null;
|
||||
}
|
||||
try (FileInputStream fis = new FileInputStream(source)) {
|
||||
return loadFromStream(fis);
|
||||
} catch (Exception e) {
|
||||
// Try as classpath resource
|
||||
InputStream resourceStream = JsonFileBarSeriesDataSource.class.getClassLoader().getResourceAsStream(source);
|
||||
if (resourceStream != null) {
|
||||
try (resourceStream) {
|
||||
return loadFromStream(resourceStream);
|
||||
} catch (Exception resourceException) {
|
||||
LOG.debug("Unable to load bars from classpath resource: {}", source, resourceException);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
LOG.debug("Unable to load bars from file: {}", source, e);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Internal implementation for loading from InputStream.
|
||||
* <p>
|
||||
* This method fully consumes the stream but does not close it, as per the
|
||||
* interface contract. The caller is responsible for closing the stream.
|
||||
*/
|
||||
private BarSeries loadFromStream(InputStream inputStream) {
|
||||
if (inputStream == null) {
|
||||
LOG.debug("Input stream is null, returning null");
|
||||
return null;
|
||||
}
|
||||
|
||||
// Read the stream fully into a String without closing it
|
||||
// This ensures we fully consume the stream while respecting the contract
|
||||
// that the caller is responsible for closing it
|
||||
String jsonContent;
|
||||
try {
|
||||
jsonContent = new String(inputStream.readAllBytes(), StandardCharsets.UTF_8);
|
||||
} catch (IOException e) {
|
||||
LOG.debug("Unable to read from input stream", e);
|
||||
return null;
|
||||
}
|
||||
|
||||
// Parse the JSON content from the String
|
||||
try (JsonReader reader = new JsonReader(new java.io.StringReader(jsonContent))) {
|
||||
return TYPEADAPTER_GSON.fromJson(reader, BarSeries.class);
|
||||
} catch (Exception e) {
|
||||
LOG.debug("Unable to load bars from JSON using TypeAdapter", e);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
}
|
||||
+978
@@ -0,0 +1,978 @@
|
||||
/*
|
||||
* SPDX-License-Identifier: MIT
|
||||
*/
|
||||
package ta4jexamples.datasources;
|
||||
|
||||
import com.google.gson.JsonArray;
|
||||
import com.google.gson.JsonObject;
|
||||
import com.google.gson.JsonParser;
|
||||
import org.apache.logging.log4j.LogManager;
|
||||
import org.apache.logging.log4j.Logger;
|
||||
import org.ta4j.core.Bar;
|
||||
import org.ta4j.core.BarSeries;
|
||||
import org.ta4j.core.BaseBarSeriesBuilder;
|
||||
import ta4jexamples.datasources.http.AbstractHttpBarSeriesDataSource;
|
||||
import ta4jexamples.datasources.http.DefaultHttpClientWrapper;
|
||||
import ta4jexamples.datasources.http.HttpClientWrapper;
|
||||
import ta4jexamples.datasources.http.HttpResponseWrapper;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.net.URI;
|
||||
import java.net.URLEncoder;
|
||||
import java.net.http.HttpClient;
|
||||
import java.net.http.HttpRequest;
|
||||
import java.net.http.HttpResponse;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.nio.file.Paths;
|
||||
import java.time.Duration;
|
||||
import java.time.Instant;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.TreeMap;
|
||||
|
||||
/**
|
||||
* Loads OHLCV data from Yahoo Finance API.
|
||||
* <p>
|
||||
* This loader fetches historical price data from Yahoo Finance's public API
|
||||
* without requiring an API key. It supports stocks, ETFs, and cryptocurrencies.
|
||||
* <p>
|
||||
* <strong>Example usage:</strong>
|
||||
*
|
||||
* <pre>
|
||||
* // Load 1 year of daily data for Apple stock (using days)
|
||||
* BarSeries series = YahooFinanceHttpBarSeriesDataSource.loadSeries("AAPL", 365);
|
||||
*
|
||||
* // Load 500 bars of hourly data for Bitcoin (using bar count)
|
||||
* BarSeries btcSeries = YahooFinanceHttpBarSeriesDataSource.loadSeries("BTC-USD", YahooFinanceInterval.HOUR_1, 500);
|
||||
*
|
||||
* // Load data for a specific date range
|
||||
* Instant start = Instant.parse("2023-01-01T00:00:00Z");
|
||||
* Instant end = Instant.parse("2023-12-31T23:59:59Z");
|
||||
* BarSeries msftSeries = YahooFinanceHttpBarSeriesDataSource.loadSeries("MSFT", YahooFinanceInterval.DAY_1, start,
|
||||
* end);
|
||||
* </pre>
|
||||
* <p>
|
||||
* <strong>Response Caching:</strong> To enable response caching for faster
|
||||
* subsequent requests, use the constructor with {@code enableResponseCaching}:
|
||||
*
|
||||
* <pre>
|
||||
* YahooFinanceHttpBarSeriesDataSource loader = new YahooFinanceHttpBarSeriesDataSource(true);
|
||||
* BarSeries series = loader.loadSeriesInstance("AAPL", YahooFinanceInterval.DAY_1, start, end);
|
||||
* </pre>
|
||||
* <p>
|
||||
* To use a custom cache directory, use the constructor with
|
||||
* {@code responseCacheDir}:
|
||||
*
|
||||
* <pre>
|
||||
* YahooFinanceHttpBarSeriesDataSource loader = new YahooFinanceHttpBarSeriesDataSource("/path/to/cache");
|
||||
* BarSeries series = loader.loadSeriesInstance("AAPL", YahooFinanceInterval.DAY_1, start, end);
|
||||
* </pre>
|
||||
* <p>
|
||||
* When caching is enabled, responses are saved to the cache directory (default:
|
||||
* {@code temp/responses}) and reused for requests within the cache validity
|
||||
* period (based on the interval). For example, daily data is cached for the
|
||||
* day, 15-minute data is cached for 15 minutes, etc. Historical data (end date
|
||||
* in the past) is cached indefinitely.
|
||||
* <p>
|
||||
* <strong>Unit Testing:</strong> For unit testing with a mock HttpClient, use
|
||||
* the constructor:
|
||||
*
|
||||
* <pre>
|
||||
* HttpClientWrapper mockHttpClient = mock(HttpClientWrapper.class);
|
||||
* YahooFinanceHttpBarSeriesDataSource loader = new YahooFinanceHttpBarSeriesDataSource(mockHttpClient);
|
||||
* // Use loader instance methods or inject into your code
|
||||
* </pre>
|
||||
* <p>
|
||||
* <strong>Note:</strong> Yahoo Finance is an unofficial API and may have rate
|
||||
* limits or availability issues. For production use, consider using official
|
||||
* APIs like Alpha Vantage, Polygon.io, or IEX Cloud.
|
||||
*
|
||||
* @since 0.20
|
||||
*/
|
||||
public class YahooFinanceHttpBarSeriesDataSource extends AbstractHttpBarSeriesDataSource {
|
||||
|
||||
public static final String YAHOO_FINANCE_API_URL = "https://query1.finance.yahoo.com/v8/finance/chart/";
|
||||
|
||||
private static final Logger LOG = LogManager.getLogger(YahooFinanceHttpBarSeriesDataSource.class);
|
||||
|
||||
@Override
|
||||
public String getSourceName() {
|
||||
return "YahooFinance";
|
||||
}
|
||||
|
||||
private static final HttpClientWrapper DEFAULT_HTTP_CLIENT = new DefaultHttpClientWrapper();
|
||||
private static final YahooFinanceHttpBarSeriesDataSource DEFAULT_INSTANCE = new YahooFinanceHttpBarSeriesDataSource(
|
||||
DEFAULT_HTTP_CLIENT);
|
||||
|
||||
/**
|
||||
* Creates a new YahooFinanceHttpBarSeriesDataSource with a default HttpClient.
|
||||
* For unit testing, use
|
||||
* {@link #YahooFinanceHttpBarSeriesDataSource(HttpClientWrapper)} to inject a
|
||||
* mock HttpClientWrapper.
|
||||
*/
|
||||
public YahooFinanceHttpBarSeriesDataSource() {
|
||||
super(DEFAULT_HTTP_CLIENT, false);
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a new YahooFinanceHttpBarSeriesDataSource with a default HttpClient
|
||||
* and caching option.
|
||||
*
|
||||
* @param enableResponseCaching if true, responses will be cached to disk for
|
||||
* faster subsequent requests
|
||||
*/
|
||||
public YahooFinanceHttpBarSeriesDataSource(boolean enableResponseCaching) {
|
||||
super(DEFAULT_HTTP_CLIENT, enableResponseCaching);
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a new YahooFinanceHttpBarSeriesDataSource with a default HttpClient
|
||||
* and custom cache directory. Response caching is automatically enabled when a
|
||||
* cache directory is specified.
|
||||
*
|
||||
* @param responseCacheDir the directory path for caching responses (can be
|
||||
* relative or absolute)
|
||||
*/
|
||||
public YahooFinanceHttpBarSeriesDataSource(String responseCacheDir) {
|
||||
super(DEFAULT_HTTP_CLIENT, responseCacheDir);
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a new YahooFinanceHttpBarSeriesDataSource with the specified
|
||||
* HttpClientWrapper. This constructor allows dependency injection of a mock
|
||||
* HttpClientWrapper for unit testing.
|
||||
*
|
||||
* @param httpClient the HttpClientWrapper to use for API requests (can be a
|
||||
* mock for testing)
|
||||
*/
|
||||
public YahooFinanceHttpBarSeriesDataSource(HttpClientWrapper httpClient) {
|
||||
super(httpClient, false);
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a new YahooFinanceHttpBarSeriesDataSource with the specified
|
||||
* HttpClientWrapper and caching option. This constructor allows dependency
|
||||
* injection of a mock HttpClientWrapper for unit testing and enables response
|
||||
* caching.
|
||||
*
|
||||
* @param httpClient the HttpClientWrapper to use for API requests
|
||||
* (can be a mock for testing)
|
||||
* @param enableResponseCaching if true, responses will be cached to disk for
|
||||
* faster subsequent requests
|
||||
*/
|
||||
public YahooFinanceHttpBarSeriesDataSource(HttpClientWrapper httpClient, boolean enableResponseCaching) {
|
||||
super(httpClient, enableResponseCaching);
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a new YahooFinanceHttpBarSeriesDataSource with the specified
|
||||
* HttpClient. This is a convenience constructor that wraps the HttpClient in a
|
||||
* DefaultHttpClientWrapper.
|
||||
*
|
||||
* @param httpClient the HttpClient to use for API requests
|
||||
*/
|
||||
public YahooFinanceHttpBarSeriesDataSource(HttpClient httpClient) {
|
||||
super(httpClient, false);
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a new YahooFinanceHttpBarSeriesDataSource with the specified
|
||||
* HttpClient and caching option.
|
||||
*
|
||||
* @param httpClient the HttpClient to use for API requests
|
||||
* @param enableResponseCaching if true, responses will be cached to disk for
|
||||
* faster subsequent requests
|
||||
*/
|
||||
public YahooFinanceHttpBarSeriesDataSource(HttpClient httpClient, boolean enableResponseCaching) {
|
||||
super(httpClient, enableResponseCaching);
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a new YahooFinanceHttpBarSeriesDataSource with the specified
|
||||
* HttpClientWrapper and custom cache directory. Response caching is
|
||||
* automatically enabled when a cache directory is specified.
|
||||
*
|
||||
* @param httpClient the HttpClientWrapper to use for API requests (can be
|
||||
* a mock for testing)
|
||||
* @param responseCacheDir the directory path for caching responses (can be
|
||||
* relative or absolute)
|
||||
*/
|
||||
public YahooFinanceHttpBarSeriesDataSource(HttpClientWrapper httpClient, String responseCacheDir) {
|
||||
super(httpClient, responseCacheDir);
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a new YahooFinanceHttpBarSeriesDataSource with the specified
|
||||
* HttpClient and custom cache directory. Response caching is automatically
|
||||
* enabled when a cache directory is specified.
|
||||
*
|
||||
* @param httpClient the HttpClient to use for API requests
|
||||
* @param responseCacheDir the directory path for caching responses (can be
|
||||
* relative or absolute)
|
||||
*/
|
||||
public YahooFinanceHttpBarSeriesDataSource(HttpClient httpClient, String responseCacheDir) {
|
||||
super(httpClient, responseCacheDir);
|
||||
}
|
||||
|
||||
/**
|
||||
* Loads historical OHLCV data for a given ticker symbol within a specified date
|
||||
* range. This is the base method that all other convenience methods delegate
|
||||
* to.
|
||||
* <p>
|
||||
* <strong>Automatic Pagination:</strong> If the requested date range exceeds
|
||||
* conservative API limits, this method automatically splits the request into
|
||||
* multiple smaller chunks, fetches them sequentially, and merges the results
|
||||
* into a single BarSeries. This ensures reliable data retrieval for large date
|
||||
* ranges while respecting API rate limits.
|
||||
* <p>
|
||||
* <strong>API Limits:</strong> Yahoo Finance's unofficial API has practical
|
||||
* limits:
|
||||
* <ul>
|
||||
* <li>Rate limits: ~2000 requests/hour per IP (may result in temporary bans if
|
||||
* exceeded)</li>
|
||||
* <li>Data range limits (approximate, may vary):
|
||||
* <ul>
|
||||
* <li>Intraday (1m-4h): Typically 60-90 days maximum per request</li>
|
||||
* <li>Daily (1d): Typically 2-5 years maximum per request</li>
|
||||
* <li>Weekly/Monthly (1wk, 1mo): Can request many years per request</li>
|
||||
* </ul>
|
||||
* </li>
|
||||
* </ul>
|
||||
* <p>
|
||||
* <strong>Conservative Limits (triggers pagination):</strong>
|
||||
* <ul>
|
||||
* <li>Intraday (1m-4h): 30 days per chunk</li>
|
||||
* <li>Hourly (1h, 4h): 60 days per chunk</li>
|
||||
* <li>Daily (1d): 1 year per chunk</li>
|
||||
* <li>Weekly/Monthly (1wk, 1mo): 5 years per chunk</li>
|
||||
* </ul>
|
||||
*
|
||||
* @param ticker the ticker symbol (e.g., "AAPL", "MSFT", "BTC-USD",
|
||||
* "ETH-USD")
|
||||
* @param interval the bar interval (must be one of the supported Yahoo
|
||||
* Finance intervals)
|
||||
* @param startDateTime the start date/time for the data range (inclusive)
|
||||
* @param endDateTime the end date/time for the data range (inclusive)
|
||||
* @return a BarSeries containing the historical data, or null if the request
|
||||
* fails
|
||||
*/
|
||||
public static BarSeries loadSeries(String ticker, YahooFinanceInterval interval, Instant startDateTime,
|
||||
Instant endDateTime) {
|
||||
return DEFAULT_INSTANCE.loadSeriesInstance(ticker, interval, startDateTime, endDateTime);
|
||||
}
|
||||
|
||||
/**
|
||||
* Loads historical OHLCV data for a given ticker symbol with a specified number
|
||||
* of bars. The end date/time is set to the current time, and the start
|
||||
* date/time is calculated based on the bar count and interval.
|
||||
* <p>
|
||||
* <strong>Note:</strong> If the calculated date range exceeds conservative API
|
||||
* limits, this method will automatically paginate the request into multiple API
|
||||
* calls and merge the results. This ensures reliable data retrieval for large
|
||||
* bar counts.
|
||||
*
|
||||
* @param ticker the ticker symbol (e.g., "AAPL", "MSFT", "BTC-USD",
|
||||
* "ETH-USD")
|
||||
* @param interval the bar interval (must be one of the supported Yahoo Finance
|
||||
* intervals)
|
||||
* @param barCount the number of bars to fetch
|
||||
* @return a BarSeries containing the historical data, or null if the request
|
||||
* fails
|
||||
*/
|
||||
public static BarSeries loadSeries(String ticker, YahooFinanceInterval interval, int barCount) {
|
||||
if (barCount <= 0) {
|
||||
LOG.error("Bar count must be greater than 0");
|
||||
return null;
|
||||
}
|
||||
|
||||
Instant endDateTime = Instant.now();
|
||||
Duration totalDuration = interval.getDuration().multipliedBy(barCount);
|
||||
Instant startDateTime = endDateTime.minus(totalDuration);
|
||||
|
||||
return loadSeries(ticker, interval, startDateTime, endDateTime);
|
||||
}
|
||||
|
||||
/**
|
||||
* Loads historical OHLCV data for a given ticker symbol with daily bars.
|
||||
* Convenience method that uses the number of days to calculate the date range.
|
||||
*
|
||||
* @param ticker the ticker symbol (e.g., "AAPL", "MSFT", "BTC-USD", "ETH-USD")
|
||||
* @param days the number of days of historical data to fetch
|
||||
* @return a BarSeries containing the historical data, or null if the request
|
||||
* fails
|
||||
*/
|
||||
public static BarSeries loadSeries(String ticker, int days) {
|
||||
return loadSeries(ticker, YahooFinanceInterval.DAY_1, days);
|
||||
}
|
||||
|
||||
/**
|
||||
* Loads historical OHLCV data for a given ticker symbol with a specified
|
||||
* interval. Convenience method that uses the number of days to calculate the
|
||||
* date range.
|
||||
*
|
||||
* @param ticker the ticker symbol (e.g., "AAPL", "MSFT", "BTC-USD",
|
||||
* "ETH-USD")
|
||||
* @param days the number of days of historical data to fetch
|
||||
* @param interval the bar interval (must be one of the supported Yahoo Finance
|
||||
* intervals)
|
||||
* @return a BarSeries containing the historical data, or null if the request
|
||||
* fails
|
||||
*/
|
||||
public static BarSeries loadSeries(String ticker, int days, YahooFinanceInterval interval) {
|
||||
if (days <= 0) {
|
||||
LOG.error("Days must be greater than 0");
|
||||
return null;
|
||||
}
|
||||
|
||||
Instant endDateTime = Instant.now();
|
||||
Instant startDateTime = endDateTime.minusSeconds(days * 86400L);
|
||||
|
||||
return loadSeries(ticker, interval, startDateTime, endDateTime);
|
||||
}
|
||||
|
||||
/**
|
||||
* Parses the Yahoo Finance API JSON response into a BarSeries.
|
||||
*/
|
||||
private static BarSeries parseYahooFinanceResponse(String jsonResponse, String ticker, Duration barInterval) {
|
||||
try {
|
||||
JsonObject root = JsonParser.parseString(jsonResponse).getAsJsonObject();
|
||||
JsonObject chart = root.getAsJsonObject("chart");
|
||||
JsonArray results = chart.getAsJsonArray("result");
|
||||
|
||||
if (results == null || results.isEmpty()) {
|
||||
LOG.error("No results found in Yahoo Finance response for ticker: {}", ticker);
|
||||
return null;
|
||||
}
|
||||
|
||||
JsonObject result = results.get(0).getAsJsonObject();
|
||||
|
||||
// Get timestamps array
|
||||
JsonArray timestamps = result.getAsJsonArray("timestamp");
|
||||
if (timestamps == null) {
|
||||
LOG.error("No timestamp data found in Yahoo Finance response for ticker: {}", ticker);
|
||||
return null;
|
||||
}
|
||||
|
||||
JsonObject indicators = result.getAsJsonObject("indicators");
|
||||
if (indicators == null) {
|
||||
LOG.error("No indicators found in Yahoo Finance response for ticker: {}", ticker);
|
||||
return null;
|
||||
}
|
||||
|
||||
JsonArray quotes = indicators.getAsJsonArray("quote");
|
||||
|
||||
if (quotes == null || quotes.isEmpty()) {
|
||||
LOG.error("No quote data found in Yahoo Finance response for ticker: {}", ticker);
|
||||
return null;
|
||||
}
|
||||
|
||||
JsonObject quote = quotes.get(0).getAsJsonObject();
|
||||
JsonArray opens = quote.getAsJsonArray("open");
|
||||
JsonArray highs = quote.getAsJsonArray("high");
|
||||
JsonArray lows = quote.getAsJsonArray("low");
|
||||
JsonArray closes = quote.getAsJsonArray("close");
|
||||
JsonArray volumes = quote.getAsJsonArray("volume");
|
||||
|
||||
BarSeries series = new BaseBarSeriesBuilder().withName(ticker).build();
|
||||
|
||||
int dataLength = timestamps.size();
|
||||
|
||||
for (int i = 0; i < dataLength; i++) {
|
||||
// Skip bars with null values
|
||||
if (timestamps.get(i).isJsonNull() || opens.get(i).isJsonNull() || highs.get(i).isJsonNull()
|
||||
|| lows.get(i).isJsonNull() || closes.get(i).isJsonNull()) {
|
||||
continue;
|
||||
}
|
||||
|
||||
long timestamp = timestamps.get(i).getAsLong();
|
||||
Instant endTime = Instant.ofEpochSecond(timestamp);
|
||||
|
||||
double openValue = opens.get(i).getAsDouble();
|
||||
double highValue = highs.get(i).getAsDouble();
|
||||
double lowValue = lows.get(i).getAsDouble();
|
||||
double closeValue = closes.get(i).getAsDouble();
|
||||
double volumeValue = volumes.get(i).isJsonNull() ? 0.0 : volumes.get(i).getAsDouble();
|
||||
|
||||
series.barBuilder()
|
||||
.timePeriod(barInterval)
|
||||
.endTime(endTime)
|
||||
.openPrice(openValue)
|
||||
.highPrice(highValue)
|
||||
.lowPrice(lowValue)
|
||||
.closePrice(closeValue)
|
||||
.volume(volumeValue)
|
||||
.amount(0)
|
||||
.add();
|
||||
}
|
||||
|
||||
LOG.debug("Successfully loaded {} bars for ticker {}", series.getBarCount(), ticker);
|
||||
return series;
|
||||
|
||||
} catch (Exception e) {
|
||||
LOG.error("Error parsing Yahoo Finance response for ticker {}: {}", ticker, e.getMessage(), e);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Merges multiple BarSeries into a single BarSeries, removing duplicates and
|
||||
* sorting chronologically. Uses a TreeMap keyed by timestamp to automatically
|
||||
* handle deduplication and sorting.
|
||||
*
|
||||
* @param chunks list of BarSeries to merge
|
||||
* @param ticker the ticker symbol (for the merged series name)
|
||||
* @param barInterval the bar interval
|
||||
* @return a merged BarSeries
|
||||
*/
|
||||
private static BarSeries mergeBarSeries(List<BarSeries> chunks, String ticker, Duration barInterval) {
|
||||
// Use TreeMap to automatically sort by timestamp and deduplicate
|
||||
TreeMap<Instant, BarData> barMap = new TreeMap<>();
|
||||
|
||||
// Collect all bars from all chunks
|
||||
for (BarSeries chunk : chunks) {
|
||||
for (int i = 0; i < chunk.getBarCount(); i++) {
|
||||
var bar = chunk.getBar(i);
|
||||
Instant endTime = bar.getEndTime();
|
||||
|
||||
// If we already have a bar at this timestamp, keep the first one (or you could
|
||||
// merge)
|
||||
barMap.putIfAbsent(endTime, new BarData(bar));
|
||||
}
|
||||
}
|
||||
|
||||
// Build the merged series
|
||||
BarSeries merged = new BaseBarSeriesBuilder().withName(ticker).build();
|
||||
|
||||
for (BarData barData : barMap.values()) {
|
||||
merged.barBuilder()
|
||||
.timePeriod(barInterval)
|
||||
.endTime(barData.endTime)
|
||||
.openPrice(barData.open)
|
||||
.highPrice(barData.high)
|
||||
.lowPrice(barData.low)
|
||||
.closePrice(barData.close)
|
||||
.volume(barData.volume)
|
||||
.amount(0)
|
||||
.add();
|
||||
}
|
||||
|
||||
LOG.debug("Merged {} chunks into {} unique bars for ticker {}", chunks.size(), merged.getBarCount(), ticker);
|
||||
return merged;
|
||||
}
|
||||
|
||||
/**
|
||||
* Instance method that loads historical OHLCV data for a given ticker symbol
|
||||
* with a specified number of bars. The end date/time is set to the current
|
||||
* time, and the start date/time is calculated based on the bar count and
|
||||
* interval.
|
||||
*
|
||||
* @param ticker the ticker symbol (e.g., "AAPL", "MSFT", "BTC-USD",
|
||||
* "ETH-USD")
|
||||
* @param interval the bar interval (must be one of the supported Yahoo Finance
|
||||
* intervals)
|
||||
* @param barCount the number of bars to fetch
|
||||
* @return a BarSeries containing the historical data, or null if the request
|
||||
* fails
|
||||
*/
|
||||
public BarSeries loadSeriesInstance(String ticker, YahooFinanceInterval interval, int barCount) {
|
||||
return loadSeriesInstance(ticker, interval, barCount, null);
|
||||
}
|
||||
|
||||
/**
|
||||
* Instance method that loads historical OHLCV data for a given ticker symbol
|
||||
* with a specified number of bars and optional notes for cache file naming. The
|
||||
* end date/time is set to the current time, and the start date/time is
|
||||
* calculated based on the bar count and interval.
|
||||
*
|
||||
* @param ticker the ticker symbol (e.g., "AAPL", "MSFT", "BTC-USD",
|
||||
* "ETH-USD")
|
||||
* @param interval the bar interval (must be one of the supported Yahoo Finance
|
||||
* intervals)
|
||||
* @param barCount the number of bars to fetch
|
||||
* @param notes optional notes to include in cache filename (for uniqueness,
|
||||
* e.g., test identifiers)
|
||||
* @return a BarSeries containing the historical data, or null if the request
|
||||
* fails
|
||||
*/
|
||||
public BarSeries loadSeriesInstance(String ticker, YahooFinanceInterval interval, int barCount, String notes) {
|
||||
if (barCount <= 0) {
|
||||
LOG.error("Bar count must be greater than 0");
|
||||
return null;
|
||||
}
|
||||
|
||||
Instant endDateTime = Instant.now();
|
||||
Duration totalDuration = interval.getDuration().multipliedBy(barCount);
|
||||
Instant startDateTime = endDateTime.minus(totalDuration);
|
||||
|
||||
return loadSeriesInstance(ticker, interval, startDateTime, endDateTime, notes);
|
||||
}
|
||||
|
||||
@Override
|
||||
public BarSeries loadSeries(String ticker, Duration interval, Instant start, Instant end) {
|
||||
if (ticker == null || ticker.trim().isEmpty()) {
|
||||
throw new IllegalArgumentException("Ticker cannot be null or empty");
|
||||
}
|
||||
if (interval == null || interval.isNegative() || interval.isZero()) {
|
||||
throw new IllegalArgumentException("Interval must be positive");
|
||||
}
|
||||
if (start == null || end == null) {
|
||||
throw new IllegalArgumentException("Start and end dates cannot be null");
|
||||
}
|
||||
if (start.isAfter(end)) {
|
||||
throw new IllegalArgumentException("Start date must be before or equal to end date");
|
||||
}
|
||||
|
||||
// Map Duration to YahooFinanceInterval
|
||||
YahooFinanceInterval yfInterval = mapDurationToInterval(interval);
|
||||
if (yfInterval == null) {
|
||||
LOG.warn("Unsupported interval duration: {}. Falling back to DAY_1", interval);
|
||||
yfInterval = YahooFinanceInterval.DAY_1;
|
||||
}
|
||||
|
||||
return loadSeriesInstance(ticker, yfInterval, start, end);
|
||||
}
|
||||
|
||||
@Override
|
||||
public BarSeries loadSeries(String source) {
|
||||
if (source == null || source.trim().isEmpty()) {
|
||||
throw new IllegalArgumentException("Source cannot be null or empty");
|
||||
}
|
||||
|
||||
// Check if it's a cache file path
|
||||
String sourcePrefix = getSourceName().isEmpty() ? "" : getSourceName() + "-";
|
||||
if (source.startsWith(responseCacheDir) || (!sourcePrefix.isEmpty() && source.contains(sourcePrefix))) {
|
||||
Path cacheFile = Paths.get(source);
|
||||
if (Files.exists(cacheFile)) {
|
||||
String cachedResponse = readFromCache(cacheFile);
|
||||
if (cachedResponse != null) {
|
||||
// Try to extract ticker from filename
|
||||
String filename = cacheFile.getFileName().toString();
|
||||
// Format: {sourceName}-TICKER-INTERVAL-START-END[_NOTES].json
|
||||
// Remove extension
|
||||
String baseName = filename.replace(".json", "");
|
||||
// Notes section is everything after the last underscore that follows the end
|
||||
// timestamp
|
||||
// We need to parse: {sourceName}-TICKER-INTERVAL-START-END[_NOTES]
|
||||
String[] parts = baseName.split("-");
|
||||
if (parts.length >= 5) {
|
||||
// Check if last part contains underscore (indicating notes section)
|
||||
// Format: END or END_NOTES
|
||||
// Notes section is ignored for parsing, so we just need to extract the ticker
|
||||
// and interval
|
||||
|
||||
String ticker = parts[1];
|
||||
// Try to determine interval from filename
|
||||
YahooFinanceInterval interval = YahooFinanceInterval.DAY_1; // Default
|
||||
try {
|
||||
interval = parseIntervalFromApiValue(parts[2]);
|
||||
} catch (IllegalArgumentException e) {
|
||||
LOG.debug("Could not parse interval from filename, using default: {}", e.getMessage());
|
||||
}
|
||||
return parseYahooFinanceResponse(cachedResponse, ticker, interval.getDuration());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// If not a cache file, return null (could be extended to parse other formats)
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Maps a Duration to the closest matching YahooFinanceInterval.
|
||||
*
|
||||
* @param duration the duration to map
|
||||
* @return the matching YahooFinanceInterval, or null if no close match is found
|
||||
*/
|
||||
private YahooFinanceInterval mapDurationToInterval(Duration duration) {
|
||||
long seconds = duration.getSeconds();
|
||||
for (YahooFinanceInterval interval : YahooFinanceInterval.values()) {
|
||||
if (interval.getDuration().getSeconds() == seconds) {
|
||||
return interval;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parses a YahooFinanceInterval from its API value string.
|
||||
*
|
||||
* @param apiValue the API value (e.g., "1m", "1d", "1wk")
|
||||
* @return the matching YahooFinanceInterval
|
||||
* @throws IllegalArgumentException if no matching interval is found
|
||||
*/
|
||||
private YahooFinanceInterval parseIntervalFromApiValue(String apiValue) {
|
||||
for (YahooFinanceInterval interval : YahooFinanceInterval.values()) {
|
||||
if (interval.getApiValue().equals(apiValue)) {
|
||||
return interval;
|
||||
}
|
||||
}
|
||||
throw new IllegalArgumentException("Unknown interval API value: " + apiValue);
|
||||
}
|
||||
|
||||
/**
|
||||
* Instance method that performs the actual loading logic. This method uses the
|
||||
* instance's HttpClient (which can be injected for testing).
|
||||
*/
|
||||
public BarSeries loadSeriesInstance(String ticker, YahooFinanceInterval interval, Instant startDateTime,
|
||||
Instant endDateTime) {
|
||||
return loadSeriesInstance(ticker, interval, startDateTime, endDateTime, null);
|
||||
}
|
||||
|
||||
/**
|
||||
* Instance method that performs the actual loading logic with optional notes.
|
||||
* This method uses the instance's HttpClient (which can be injected for
|
||||
* testing).
|
||||
*
|
||||
* @param ticker the ticker symbol
|
||||
* @param interval the interval
|
||||
* @param startDateTime the start date/time
|
||||
* @param endDateTime the end date/time
|
||||
* @param notes optional notes to include in cache filename (for
|
||||
* uniqueness)
|
||||
* @return the BarSeries or null if request fails
|
||||
*/
|
||||
public BarSeries loadSeriesInstance(String ticker, YahooFinanceInterval interval, Instant startDateTime,
|
||||
Instant endDateTime, String notes) {
|
||||
if (ticker == null || ticker.trim().isEmpty()) {
|
||||
LOG.error("Ticker symbol cannot be null or empty");
|
||||
return null;
|
||||
}
|
||||
|
||||
if (startDateTime == null || endDateTime == null) {
|
||||
LOG.error("Start and end date/time cannot be null");
|
||||
return null;
|
||||
}
|
||||
|
||||
if (startDateTime.isAfter(endDateTime)) {
|
||||
LOG.error("Start date/time must be before or equal to end date/time");
|
||||
return null;
|
||||
}
|
||||
|
||||
Duration requestedRange = Duration.between(startDateTime, endDateTime);
|
||||
Duration conservativeLimit = this.getConservativeLimit(interval);
|
||||
|
||||
// If the requested range exceeds conservative limits, paginate the request
|
||||
if (requestedRange.compareTo(conservativeLimit) > 0) {
|
||||
LOG.debug(
|
||||
"Requested date range ({}) exceeds conservative limit ({}) for interval {}. "
|
||||
+ "Splitting into multiple requests and combining results.",
|
||||
requestedRange, conservativeLimit, interval);
|
||||
return loadSeriesPaginated(ticker, interval, startDateTime, endDateTime, conservativeLimit, notes);
|
||||
}
|
||||
|
||||
// Single request for smaller ranges
|
||||
return loadSeriesSingleRequest(ticker, interval, startDateTime, endDateTime, notes);
|
||||
}
|
||||
|
||||
/**
|
||||
* Truncates a timestamp based on the interval to enable cache hits for requests
|
||||
* within the same time period. This override provides Yahoo Finance-specific
|
||||
* truncation logic for week and month boundaries.
|
||||
* <p>
|
||||
* For example, for 1-day intervals, timestamps are truncated to the start of
|
||||
* the day. For 15-minute intervals, timestamps are truncated to the start of
|
||||
* the 15-minute period. For weekly intervals, timestamps are truncated to the
|
||||
* start of the week (Monday). For monthly intervals, timestamps are truncated
|
||||
* to the start of the month.
|
||||
*
|
||||
* @param instant the timestamp to truncate
|
||||
* @param interval the interval to use for truncation
|
||||
* @return the truncated timestamp
|
||||
*/
|
||||
|
||||
/**
|
||||
* Generates the cache file path for a given request.
|
||||
*
|
||||
* @param ticker the ticker symbol
|
||||
* @param interval the interval
|
||||
* @param startDateTime the start date/time (will be truncated)
|
||||
* @param endDateTime the end date/time (will be truncated)
|
||||
* @param notes optional notes section to append to filename (can be
|
||||
* null or empty)
|
||||
* @return the cache file path
|
||||
*/
|
||||
private Path getCacheFilePath(String ticker, YahooFinanceInterval interval, Instant startDateTime,
|
||||
Instant endDateTime, String notes) {
|
||||
return getCacheFilePath(ticker, startDateTime, endDateTime, interval.getDuration(), notes);
|
||||
}
|
||||
|
||||
/**
|
||||
* Generates the cache file path for a given request (without notes).
|
||||
*
|
||||
* @param ticker the ticker symbol
|
||||
* @param interval the interval
|
||||
* @param startDateTime the start date/time (will be truncated)
|
||||
* @param endDateTime the end date/time (will be truncated)
|
||||
* @return the cache file path
|
||||
*/
|
||||
private Path getCacheFilePath(String ticker, YahooFinanceInterval interval, Instant startDateTime,
|
||||
Instant endDateTime) {
|
||||
return getCacheFilePath(ticker, interval, startDateTime, endDateTime, null);
|
||||
}
|
||||
|
||||
/**
|
||||
* Makes a single API request for the specified date range with optional notes.
|
||||
* This is used for requests that don't exceed conservative limits. If caching
|
||||
* is enabled, checks cache first before making the API request.
|
||||
*
|
||||
* @param ticker the ticker symbol
|
||||
* @param interval the interval
|
||||
* @param startDateTime the start date/time
|
||||
* @param endDateTime the end date/time
|
||||
* @param notes optional notes to include in cache filename (for
|
||||
* uniqueness)
|
||||
* @return the BarSeries or null if request fails
|
||||
*/
|
||||
private BarSeries loadSeriesSingleRequest(String ticker, YahooFinanceInterval interval, Instant startDateTime,
|
||||
Instant endDateTime, String notes) {
|
||||
// Check cache first if caching is enabled
|
||||
if (enableResponseCaching) {
|
||||
// Try exact match first (with or without notes)
|
||||
Path cacheFile = getCacheFilePath(ticker, interval, startDateTime, endDateTime, notes);
|
||||
if (isCacheValid(cacheFile, interval.getDuration(), endDateTime)) {
|
||||
String cachedResponse = readFromCache(cacheFile);
|
||||
if (cachedResponse != null) {
|
||||
LOG.debug("Using cached response for {} ({} to {})", ticker, startDateTime, endDateTime);
|
||||
return parseYahooFinanceResponse(cachedResponse, ticker, interval.getDuration());
|
||||
}
|
||||
}
|
||||
// Also try without notes (for backward compatibility)
|
||||
if (notes != null && !notes.trim().isEmpty()) {
|
||||
Path cacheFileNoNotes = getCacheFilePath(ticker, interval, startDateTime, endDateTime);
|
||||
if (isCacheValid(cacheFileNoNotes, interval.getDuration(), endDateTime)) {
|
||||
String cachedResponse = readFromCache(cacheFileNoNotes);
|
||||
if (cachedResponse != null) {
|
||||
LOG.debug("Using cached response for {} ({} to {})", ticker, startDateTime, endDateTime);
|
||||
return parseYahooFinanceResponse(cachedResponse, ticker, interval.getDuration());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
String encodedTicker = URLEncoder.encode(ticker.trim(), StandardCharsets.UTF_8);
|
||||
long period1 = startDateTime.getEpochSecond();
|
||||
long period2 = endDateTime.getEpochSecond();
|
||||
|
||||
String url = String.format("%s%s?interval=%s&period1=%d&period2=%d", YAHOO_FINANCE_API_URL, encodedTicker,
|
||||
interval.getApiValue(), period1, period2);
|
||||
|
||||
LOG.trace("Fetching data from Yahoo Finance: {}", url);
|
||||
|
||||
HttpRequest request = HttpRequest.newBuilder()
|
||||
.uri(URI.create(url))
|
||||
.header("User-Agent", "Mozilla/5.0")
|
||||
.timeout(Duration.ofSeconds(30))
|
||||
.GET()
|
||||
.build();
|
||||
|
||||
HttpResponseWrapper<String> response = httpClient.send(request, HttpResponse.BodyHandlers.ofString());
|
||||
|
||||
if (response.statusCode() != 200) {
|
||||
LOG.error("Yahoo Finance API returned status code: {}", response.statusCode());
|
||||
return null;
|
||||
}
|
||||
|
||||
String responseBody = response.body();
|
||||
LOG.trace("Response body: {}", responseBody);
|
||||
|
||||
// Cache the response if caching is enabled
|
||||
if (enableResponseCaching) {
|
||||
Path cacheFile = getCacheFilePath(ticker, interval, startDateTime, endDateTime, notes);
|
||||
writeToCache(cacheFile, responseBody);
|
||||
}
|
||||
|
||||
return parseYahooFinanceResponse(responseBody, ticker, interval.getDuration());
|
||||
|
||||
} catch (IOException | InterruptedException e) {
|
||||
LOG.error("Error fetching data from Yahoo Finance for ticker {}: {}", ticker, e.getMessage(), e);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Loads data by splitting a large date range into multiple smaller requests
|
||||
* (pagination). Each chunk respects the conservative limit, and results are
|
||||
* merged chronologically.
|
||||
*
|
||||
* @param ticker the ticker symbol
|
||||
* @param interval the bar interval
|
||||
* @param startDateTime the start date/time
|
||||
* @param endDateTime the end date/time
|
||||
* @param chunkSize the maximum size for each chunk
|
||||
* @param notes optional notes to include in cache filename (for
|
||||
* uniqueness)
|
||||
* @return a BarSeries containing all merged data, or null if all requests fail
|
||||
*/
|
||||
private BarSeries loadSeriesPaginated(String ticker, YahooFinanceInterval interval, Instant startDateTime,
|
||||
Instant endDateTime, Duration chunkSize, String notes) {
|
||||
List<BarSeries> chunks = new ArrayList<>();
|
||||
Instant currentStart = startDateTime;
|
||||
int requestCount = 0;
|
||||
|
||||
// Calculate number of chunks needed
|
||||
Duration totalRange = Duration.between(startDateTime, endDateTime);
|
||||
int estimatedChunks = (int) Math.ceil((double) totalRange.toSeconds() / chunkSize.toSeconds());
|
||||
LOG.trace("Splitting request into approximately {} chunks", estimatedChunks);
|
||||
|
||||
while (currentStart.isBefore(endDateTime)) {
|
||||
// Calculate chunk end time (don't exceed the requested end time)
|
||||
Instant chunkEnd = currentStart.plus(chunkSize);
|
||||
if (chunkEnd.isAfter(endDateTime)) {
|
||||
chunkEnd = endDateTime;
|
||||
}
|
||||
|
||||
requestCount++;
|
||||
LOG.trace("Fetching chunk {}/? ({} to {})", requestCount, currentStart, chunkEnd);
|
||||
|
||||
BarSeries chunk = loadSeriesSingleRequest(ticker, interval, currentStart, chunkEnd, notes);
|
||||
if (chunk != null && chunk.getBarCount() > 0) {
|
||||
chunks.add(chunk);
|
||||
LOG.trace("Successfully loaded chunk {} with {} bars", requestCount, chunk.getBarCount());
|
||||
} else {
|
||||
LOG.warn("Chunk {} returned no data or failed", requestCount);
|
||||
}
|
||||
|
||||
// Move to next chunk (start from the end of current chunk)
|
||||
currentStart = chunkEnd;
|
||||
|
||||
// If we've reached the end, break
|
||||
if (chunkEnd.equals(endDateTime) || !currentStart.isBefore(endDateTime)) {
|
||||
break;
|
||||
}
|
||||
|
||||
// Add a small delay between requests to avoid rate limiting
|
||||
try {
|
||||
Thread.sleep(100); // 100ms delay between requests
|
||||
} catch (InterruptedException e) {
|
||||
Thread.currentThread().interrupt();
|
||||
LOG.warn("Interrupted during pagination delay");
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (chunks.isEmpty()) {
|
||||
LOG.error("All paginated requests failed for ticker {}", ticker);
|
||||
return null;
|
||||
}
|
||||
|
||||
LOG.debug("Successfully fetched {} chunks, merging {} total bars", chunks.size(),
|
||||
chunks.stream().mapToInt(BarSeries::getBarCount).sum());
|
||||
|
||||
return mergeBarSeries(chunks, ticker, interval.getDuration());
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the conservative (safe) maximum date range for a given interval.
|
||||
* These are smaller than the absolute maximums to ensure reliable API
|
||||
* responses. Used to determine when pagination is needed.
|
||||
* <p>
|
||||
* This method is protected to allow subclasses (e.g., in tests) to override the
|
||||
* conservative limit for testing pagination functionality.
|
||||
*
|
||||
* @param interval the bar interval
|
||||
* @return the conservative maximum date range
|
||||
*/
|
||||
protected Duration getConservativeLimit(YahooFinanceInterval interval) {
|
||||
return switch (interval) {
|
||||
case MINUTE_1, MINUTE_5, MINUTE_15, MINUTE_30 -> Duration.ofDays(30); // 30 days for intraday (conservative)
|
||||
case HOUR_1, HOUR_4 -> Duration.ofDays(60); // 60 days for hourly (conservative)
|
||||
case DAY_1 -> Duration.ofDays(365); // 1 year for daily (conservative)
|
||||
case WEEK_1, MONTH_1 -> Duration.ofDays(365 * 5); // 5 years for weekly/monthly (conservative)
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Supported intervals for Yahoo Finance API. These correspond to the intervals
|
||||
* that Yahoo Finance's chart API supports.
|
||||
*/
|
||||
public enum YahooFinanceInterval {
|
||||
/**
|
||||
* 1 minute bars
|
||||
*/
|
||||
MINUTE_1(Duration.ofMinutes(1), "1m"),
|
||||
/**
|
||||
* 5 minute bars
|
||||
*/
|
||||
MINUTE_5(Duration.ofMinutes(5), "5m"),
|
||||
/**
|
||||
* 15 minute bars
|
||||
*/
|
||||
MINUTE_15(Duration.ofMinutes(15), "15m"),
|
||||
/**
|
||||
* 30 minute bars
|
||||
*/
|
||||
MINUTE_30(Duration.ofMinutes(30), "30m"),
|
||||
/**
|
||||
* 1 hour bars
|
||||
*/
|
||||
HOUR_1(Duration.ofHours(1), "1h"),
|
||||
/**
|
||||
* 4 hour bars
|
||||
*/
|
||||
HOUR_4(Duration.ofHours(4), "4h"),
|
||||
/**
|
||||
* 1 day bars
|
||||
*/
|
||||
DAY_1(Duration.ofDays(1), "1d"),
|
||||
/**
|
||||
* 1 week bars
|
||||
*/
|
||||
WEEK_1(Duration.ofDays(7), "1wk"),
|
||||
/**
|
||||
* 1 month bars
|
||||
*/
|
||||
MONTH_1(Duration.ofDays(30), "1mo");
|
||||
|
||||
private final Duration duration;
|
||||
private final String apiValue;
|
||||
|
||||
YahooFinanceInterval(Duration duration, String apiValue) {
|
||||
this.duration = duration;
|
||||
this.apiValue = apiValue;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the Duration for this interval.
|
||||
*
|
||||
* @return the Duration
|
||||
*/
|
||||
public Duration getDuration() {
|
||||
return duration;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the API string value for this interval.
|
||||
*
|
||||
* @return the API string value
|
||||
*/
|
||||
public String getApiValue() {
|
||||
return apiValue;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Helper class to hold bar data during merging.
|
||||
*/
|
||||
private static class BarData {
|
||||
final Instant endTime;
|
||||
final double open;
|
||||
final double high;
|
||||
final double low;
|
||||
final double close;
|
||||
final double volume;
|
||||
|
||||
BarData(Bar bar) {
|
||||
this.endTime = bar.getEndTime();
|
||||
this.open = bar.getOpenPrice().doubleValue();
|
||||
this.high = bar.getHighPrice().doubleValue();
|
||||
this.low = bar.getLowPrice().doubleValue();
|
||||
this.close = bar.getClosePrice().doubleValue();
|
||||
this.volume = bar.getVolume().doubleValue();
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
+247
@@ -0,0 +1,247 @@
|
||||
/*
|
||||
* SPDX-License-Identifier: MIT
|
||||
*/
|
||||
package ta4jexamples.datasources.file;
|
||||
|
||||
import org.apache.logging.log4j.LogManager;
|
||||
import org.apache.logging.log4j.Logger;
|
||||
import org.ta4j.core.BarSeries;
|
||||
import org.ta4j.core.BaseBarSeriesBuilder;
|
||||
import ta4jexamples.datasources.BarSeriesDataSource;
|
||||
|
||||
import java.time.Duration;
|
||||
import java.time.Instant;
|
||||
import java.time.ZoneOffset;
|
||||
import java.time.format.DateTimeFormatter;
|
||||
|
||||
/**
|
||||
* Abstract base class for file-based BarSeries data sources.
|
||||
* <p>
|
||||
* This class provides common infrastructure for file-based data sources,
|
||||
* including:
|
||||
* <ul>
|
||||
* <li>Parameter validation for ticker, interval, and date range</li>
|
||||
* <li>Filename pattern building based on interval and date range</li>
|
||||
* <li>Date range filtering for BarSeries</li>
|
||||
* <li>DateTimeFormatter utilities for filename formatting</li>
|
||||
* </ul>
|
||||
* <p>
|
||||
* Subclasses should implement the file format-specific logic such as:
|
||||
* <ul>
|
||||
* <li>Loading and parsing files (CSV, JSON, etc.)</li>
|
||||
* <li>File searching with format-specific patterns</li>
|
||||
* <li>Format-specific data transformation</li>
|
||||
* </ul>
|
||||
* <p>
|
||||
* <strong>Example usage:</strong>
|
||||
*
|
||||
* <pre>
|
||||
* public class MyFileDataSource extends AbstractFileBarSeriesDataSource {
|
||||
* public MyFileDataSource() {
|
||||
* super("MySource");
|
||||
* }
|
||||
*
|
||||
* // Implement abstract methods for format-specific loading
|
||||
* }
|
||||
* </pre>
|
||||
*
|
||||
* @since 0.20
|
||||
*/
|
||||
public abstract class AbstractFileBarSeriesDataSource implements BarSeriesDataSource {
|
||||
|
||||
private static final Logger LOG = LogManager.getLogger(AbstractFileBarSeriesDataSource.class);
|
||||
|
||||
/**
|
||||
* Date format for filenames: yyyyMMdd
|
||||
*/
|
||||
protected static final DateTimeFormatter FILENAME_DATE_FORMAT = DateTimeFormatter.ofPattern("yyyyMMdd");
|
||||
|
||||
/**
|
||||
* Date-time format for filenames with hours: yyyyMMddHH
|
||||
*/
|
||||
protected static final DateTimeFormatter FILENAME_DATETIME_HOUR_FORMAT = DateTimeFormatter.ofPattern("yyyyMMddHH");
|
||||
|
||||
/**
|
||||
* Date-time format for filenames with minutes: yyyyMMddHHmm
|
||||
*/
|
||||
protected static final DateTimeFormatter FILENAME_DATETIME_MINUTE_FORMAT = DateTimeFormatter
|
||||
.ofPattern("yyyyMMddHHmm");
|
||||
|
||||
private final String sourceName;
|
||||
|
||||
/**
|
||||
* Creates a new AbstractFileBarSeriesDataSource with the specified source name.
|
||||
*
|
||||
* @param sourceName the source name for building file search patterns (e.g.,
|
||||
* "Bitstamp", "Coinbase"). Use empty string for generic
|
||||
* sources without a prefix.
|
||||
*/
|
||||
protected AbstractFileBarSeriesDataSource(String sourceName) {
|
||||
this.sourceName = sourceName != null ? sourceName : "";
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getSourceName() {
|
||||
return sourceName;
|
||||
}
|
||||
|
||||
@Override
|
||||
public BarSeries loadSeries(String ticker, Duration interval, Instant start, Instant end) {
|
||||
validateParameters(ticker, interval, start, end);
|
||||
|
||||
// Build search patterns for filename matching
|
||||
// Standard pattern:
|
||||
// {sourcePrefix}{ticker}-{interval}-{startDateTime}_{endDateTime}.{extension}
|
||||
// DateTime format depends on interval: minutes -> HHmm, hours -> HH, days ->
|
||||
// date only
|
||||
DateTimeFormatter dateTimeFormatter = getDateTimeFormatterForInterval(interval);
|
||||
String startDateTimeStr = start.atZone(ZoneOffset.UTC).format(dateTimeFormatter);
|
||||
String endDateTimeStr = end.atZone(ZoneOffset.UTC).format(dateTimeFormatter);
|
||||
|
||||
// Also prepare date-only format since existing files use this format
|
||||
String startDateStr = start.atZone(ZoneOffset.UTC).format(FILENAME_DATE_FORMAT);
|
||||
String endDateStr = end.atZone(ZoneOffset.UTC).format(FILENAME_DATE_FORMAT);
|
||||
|
||||
String intervalStr = formatIntervalForFilename(interval);
|
||||
String sourcePrefix = sourceName.isEmpty() ? "" : sourceName + "-";
|
||||
|
||||
// Delegate to subclass for format-specific file searching
|
||||
BarSeries series = searchAndLoadFile(ticker, intervalStr, sourcePrefix, startDateTimeStr, endDateTimeStr,
|
||||
startDateStr, endDateStr, interval, start, end);
|
||||
|
||||
if (series != null && !series.isEmpty()) {
|
||||
return series;
|
||||
}
|
||||
|
||||
LOG.debug("No {} file found matching ticker: {}, interval: {}, date range: {} to {}", getFileExtension(),
|
||||
ticker, interval, start, end);
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Validates the parameters for loading a series.
|
||||
*
|
||||
* @param ticker the ticker symbol
|
||||
* @param interval the bar interval
|
||||
* @param start the start date/time
|
||||
* @param end the end date/time
|
||||
* @throws IllegalArgumentException if any parameter is invalid
|
||||
*/
|
||||
protected void validateParameters(String ticker, Duration interval, Instant start, Instant end) {
|
||||
if (ticker == null || ticker.trim().isEmpty()) {
|
||||
throw new IllegalArgumentException("Ticker cannot be null or empty");
|
||||
}
|
||||
if (interval == null || interval.isNegative() || interval.isZero()) {
|
||||
throw new IllegalArgumentException("Interval must be positive");
|
||||
}
|
||||
if (start == null || end == null) {
|
||||
throw new IllegalArgumentException("Start and end dates cannot be null");
|
||||
}
|
||||
if (start.isAfter(end)) {
|
||||
throw new IllegalArgumentException("Start date must be before or equal to end date");
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Searches for and loads a file matching the specified patterns. Subclasses
|
||||
* should implement this method to handle format-specific file searching logic.
|
||||
* <p>
|
||||
* This method is called by
|
||||
* {@link #loadSeries(String, Duration, Instant, Instant)} with pre-formatted
|
||||
* date strings and interval strings. Subclasses should try multiple patterns
|
||||
* (exact match, date-only format, broader patterns) and return the first
|
||||
* matching file.
|
||||
*
|
||||
* @param ticker the ticker symbol (already validated)
|
||||
* @param intervalStr the formatted interval string (e.g., "PT1D", "PT5M")
|
||||
* @param sourcePrefix the source prefix (e.g., "Bitstamp-", or empty
|
||||
* string)
|
||||
* @param startDateTimeStr the formatted start date-time string
|
||||
* @param endDateTimeStr the formatted end date-time string
|
||||
* @param startDateStr the formatted start date string (date only)
|
||||
* @param endDateStr the formatted end date string (date only)
|
||||
* @param interval the interval duration
|
||||
* @param start the start date/time
|
||||
* @param end the end date/time
|
||||
* @return the loaded BarSeries, or null if no matching file is found
|
||||
*/
|
||||
protected abstract BarSeries searchAndLoadFile(String ticker, String intervalStr, String sourcePrefix,
|
||||
String startDateTimeStr, String endDateTimeStr, String startDateStr, String endDateStr, Duration interval,
|
||||
Instant start, Instant end);
|
||||
|
||||
/**
|
||||
* Returns the file extension (without the dot) for this data source. Used in
|
||||
* log messages and file pattern building.
|
||||
*
|
||||
* @return the file extension (e.g., "csv", "json")
|
||||
*/
|
||||
protected abstract String getFileExtension();
|
||||
|
||||
/**
|
||||
* Determines the appropriate DateTimeFormatter for filename datetime formatting
|
||||
* based on the interval. For minute-level intervals, includes hours and
|
||||
* minutes. For hour-level intervals, includes hours. For day-level intervals,
|
||||
* uses date only.
|
||||
*
|
||||
* @param interval the bar interval
|
||||
* @return the appropriate DateTimeFormatter
|
||||
*/
|
||||
protected DateTimeFormatter getDateTimeFormatterForInterval(Duration interval) {
|
||||
long seconds = interval.getSeconds();
|
||||
if (seconds < 3600) {
|
||||
// Interval is in minutes or seconds - include hours and minutes
|
||||
return FILENAME_DATETIME_MINUTE_FORMAT;
|
||||
} else if (seconds < 86400) {
|
||||
// Interval is in hours - include hours
|
||||
return FILENAME_DATETIME_HOUR_FORMAT;
|
||||
} else {
|
||||
// Interval is in days or longer - date only
|
||||
return FILENAME_DATE_FORMAT;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Formats a Duration as an ISO 8601 interval string for use in filenames.
|
||||
*
|
||||
* @param interval the duration to format
|
||||
* @return the ISO 8601 duration string (e.g., "PT1D", "PT5M", "PT1H")
|
||||
*/
|
||||
protected String formatIntervalForFilename(Duration interval) {
|
||||
long seconds = interval.getSeconds();
|
||||
if (seconds % 86400 == 0) {
|
||||
return "PT" + (seconds / 86400) + "D";
|
||||
} else if (seconds % 3600 == 0) {
|
||||
return "PT" + (seconds / 3600) + "H";
|
||||
} else if (seconds % 60 == 0) {
|
||||
return "PT" + (seconds / 60) + "M";
|
||||
} else {
|
||||
return "PT" + seconds + "S";
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Filters a BarSeries to only include bars within the specified date range.
|
||||
*
|
||||
* @param series the series to filter
|
||||
* @param start the start date (inclusive)
|
||||
* @param end the end date (inclusive)
|
||||
* @return a new BarSeries containing only bars within the date range, or null
|
||||
* if no bars match
|
||||
*/
|
||||
protected BarSeries filterSeriesByDateRange(BarSeries series, Instant start, Instant end) {
|
||||
if (series == null || series.isEmpty()) {
|
||||
return null;
|
||||
}
|
||||
|
||||
var filteredSeries = new BaseBarSeriesBuilder().withName(series.getName()).build();
|
||||
for (int i = 0; i < series.getBarCount(); i++) {
|
||||
var bar = series.getBar(i);
|
||||
Instant barEnd = bar.getEndTime();
|
||||
if (!barEnd.isBefore(start) && !barEnd.isAfter(end)) {
|
||||
filteredSeries.addBar(bar);
|
||||
}
|
||||
}
|
||||
|
||||
return filteredSeries.isEmpty() ? null : filteredSeries;
|
||||
}
|
||||
}
|
||||
+430
@@ -0,0 +1,430 @@
|
||||
/*
|
||||
* SPDX-License-Identifier: MIT
|
||||
*/
|
||||
package ta4jexamples.datasources.http;
|
||||
|
||||
import org.apache.logging.log4j.LogManager;
|
||||
import org.apache.logging.log4j.Logger;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.net.http.HttpClient;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.nio.file.Paths;
|
||||
import java.time.Duration;
|
||||
import java.time.Instant;
|
||||
import java.time.ZoneOffset;
|
||||
import java.time.ZonedDateTime;
|
||||
|
||||
/**
|
||||
* Abstract base class for HTTP-based BarSeries data sources.
|
||||
* <p>
|
||||
* This class provides common infrastructure for HTTP-based data sources,
|
||||
* including:
|
||||
* <ul>
|
||||
* <li>HTTP client management via {@link HttpClientWrapper}</li>
|
||||
* <li>Response caching to disk for faster subsequent requests</li>
|
||||
* <li>Cache validation and management</li>
|
||||
* <li>Common cache file operations</li>
|
||||
* </ul>
|
||||
* <p>
|
||||
* Subclasses should implement the API-specific logic such as:
|
||||
* <ul>
|
||||
* <li>Building API-specific URLs and request parameters</li>
|
||||
* <li>Parsing API-specific response formats</li>
|
||||
* <li>Defining interval enums and their API values</li>
|
||||
* <li>Implementing interval-specific timestamp truncation logic</li>
|
||||
* </ul>
|
||||
* <p>
|
||||
* <strong>Example usage:</strong>
|
||||
*
|
||||
* <pre>
|
||||
* public class MyHttpDataSource extends AbstractHttpBarSeriesDataSource {
|
||||
* public MyHttpDataSource() {
|
||||
* super(new DefaultHttpClientWrapper(), false);
|
||||
* }
|
||||
*
|
||||
* // Implement abstract methods and API-specific logic
|
||||
* }
|
||||
* </pre>
|
||||
*
|
||||
* @since 0.20
|
||||
*/
|
||||
public abstract class AbstractHttpBarSeriesDataSource implements HttpBarSeriesDataSource {
|
||||
|
||||
/**
|
||||
* Default directory for caching HTTP responses.
|
||||
*/
|
||||
public static final String DEFAULT_RESPONSE_CACHE_DIR = "temp/responses";
|
||||
|
||||
private static final Logger LOG = LogManager.getLogger(AbstractHttpBarSeriesDataSource.class);
|
||||
|
||||
protected final HttpClientWrapper httpClient;
|
||||
protected final boolean enableResponseCaching;
|
||||
protected final String responseCacheDir;
|
||||
|
||||
/**
|
||||
* Creates a new AbstractHttpBarSeriesDataSource with the specified
|
||||
* HttpClientWrapper and caching option, using the default cache directory.
|
||||
*
|
||||
* @param httpClient the HttpClientWrapper to use for API requests
|
||||
* (can be a mock for testing)
|
||||
* @param enableResponseCaching if true, responses will be cached to disk for
|
||||
* faster subsequent requests
|
||||
* @throws IllegalArgumentException if httpClient is null
|
||||
*/
|
||||
protected AbstractHttpBarSeriesDataSource(HttpClientWrapper httpClient, boolean enableResponseCaching) {
|
||||
if (httpClient == null) {
|
||||
throw new IllegalArgumentException("HttpClientWrapper cannot be null");
|
||||
}
|
||||
this.httpClient = httpClient;
|
||||
this.enableResponseCaching = enableResponseCaching;
|
||||
this.responseCacheDir = DEFAULT_RESPONSE_CACHE_DIR;
|
||||
if (enableResponseCaching) {
|
||||
ensureCacheDirectoryExists();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a new AbstractHttpBarSeriesDataSource with the specified
|
||||
* HttpClientWrapper and custom cache directory. Response caching is
|
||||
* automatically enabled when a cache directory is specified.
|
||||
*
|
||||
* @param httpClient the HttpClientWrapper to use for API requests (can be
|
||||
* a mock for testing)
|
||||
* @param responseCacheDir the directory path for caching responses (can be
|
||||
* relative or absolute)
|
||||
* @throws IllegalArgumentException if httpClient is null or responseCacheDir is
|
||||
* null or empty
|
||||
*/
|
||||
protected AbstractHttpBarSeriesDataSource(HttpClientWrapper httpClient, String responseCacheDir) {
|
||||
if (httpClient == null) {
|
||||
throw new IllegalArgumentException("HttpClientWrapper cannot be null");
|
||||
}
|
||||
if (responseCacheDir == null || responseCacheDir.trim().isEmpty()) {
|
||||
throw new IllegalArgumentException("Response cache directory cannot be null or empty");
|
||||
}
|
||||
this.httpClient = httpClient;
|
||||
this.enableResponseCaching = true;
|
||||
this.responseCacheDir = responseCacheDir.trim();
|
||||
ensureCacheDirectoryExists();
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a new AbstractHttpBarSeriesDataSource with the specified HttpClient.
|
||||
* This is a convenience constructor that wraps the HttpClient in a
|
||||
* DefaultHttpClientWrapper, using the default cache directory.
|
||||
*
|
||||
* @param httpClient the HttpClient to use for API requests
|
||||
* @param enableResponseCaching if true, responses will be cached to disk for
|
||||
* faster subsequent requests
|
||||
*/
|
||||
protected AbstractHttpBarSeriesDataSource(HttpClient httpClient, boolean enableResponseCaching) {
|
||||
this(new DefaultHttpClientWrapper(httpClient), enableResponseCaching);
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a new AbstractHttpBarSeriesDataSource with the specified HttpClient
|
||||
* and custom cache directory. This is a convenience constructor that wraps the
|
||||
* HttpClient in a DefaultHttpClientWrapper. Response caching is automatically
|
||||
* enabled when a cache directory is specified.
|
||||
*
|
||||
* @param httpClient the HttpClient to use for API requests
|
||||
* @param responseCacheDir the directory path for caching responses (can be
|
||||
* relative or absolute)
|
||||
*/
|
||||
protected AbstractHttpBarSeriesDataSource(HttpClient httpClient, String responseCacheDir) {
|
||||
this(new DefaultHttpClientWrapper(httpClient), responseCacheDir);
|
||||
}
|
||||
|
||||
@Override
|
||||
public HttpClientWrapper getHttpClient() {
|
||||
return httpClient;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean isResponseCachingEnabled() {
|
||||
return enableResponseCaching;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the response cache directory path.
|
||||
*
|
||||
* @return the cache directory path
|
||||
*/
|
||||
public String getResponseCacheDir() {
|
||||
return responseCacheDir;
|
||||
}
|
||||
|
||||
/**
|
||||
* Ensures the cache directory exists, creating it if necessary.
|
||||
*/
|
||||
protected void ensureCacheDirectoryExists() {
|
||||
try {
|
||||
Path cacheDir = Paths.get(responseCacheDir);
|
||||
if (!Files.exists(cacheDir)) {
|
||||
Files.createDirectories(cacheDir);
|
||||
LOG.debug("Created cache directory: {}", cacheDir.toAbsolutePath());
|
||||
}
|
||||
} catch (IOException e) {
|
||||
LOG.warn("Failed to create cache directory: {}", e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Truncates a timestamp based on the interval duration to enable cache hits for
|
||||
* requests within the same time period. This is a generic implementation that
|
||||
* works with Duration. Subclasses can override this method to provide more
|
||||
* precise truncation logic for specific interval types (e.g., week boundaries,
|
||||
* month boundaries).
|
||||
* <p>
|
||||
* For example, for 1-day intervals, timestamps are truncated to the start of
|
||||
* the day. For 15-minute intervals, timestamps are truncated to the start of
|
||||
* the 15-minute period.
|
||||
*
|
||||
* @param instant the timestamp to truncate
|
||||
* @param interval the interval duration to use for truncation
|
||||
* @return the truncated timestamp
|
||||
*/
|
||||
protected Instant truncateTimestampForCache(Instant instant, Duration interval) {
|
||||
ZonedDateTime zdt = instant.atZone(ZoneOffset.UTC);
|
||||
ZonedDateTime truncated;
|
||||
|
||||
long seconds = interval.getSeconds();
|
||||
long minutes = seconds / 60;
|
||||
long hours = minutes / 60;
|
||||
long days = hours / 24;
|
||||
|
||||
if (days >= 30) {
|
||||
// For monthly intervals (approximately 30 days), truncate to start of month
|
||||
truncated = zdt.withDayOfMonth(1).withHour(0).withMinute(0).withSecond(0).withNano(0);
|
||||
} else if (days >= 7) {
|
||||
// For weekly intervals (approximately 7 days), truncate to start of week
|
||||
// (Monday)
|
||||
int daysFromMonday = zdt.getDayOfWeek().getValue() - java.time.DayOfWeek.MONDAY.getValue();
|
||||
if (daysFromMonday < 0) {
|
||||
daysFromMonday += 7; // Handle Sunday (value 7)
|
||||
}
|
||||
truncated = zdt.minusDays(daysFromMonday).withHour(0).withMinute(0).withSecond(0).withNano(0);
|
||||
} else if (days >= 1) {
|
||||
// For daily intervals, truncate to start of day
|
||||
truncated = zdt.withHour(0).withMinute(0).withSecond(0).withNano(0);
|
||||
} else if (hours >= 1) {
|
||||
// For hourly intervals, truncate to start of hour period
|
||||
int hour = (int) ((zdt.getHour() / hours) * hours);
|
||||
truncated = zdt.withHour(hour).withMinute(0).withSecond(0).withNano(0);
|
||||
} else if (minutes >= 1) {
|
||||
// For minute intervals, truncate to start of minute period
|
||||
int minute = (int) ((zdt.getMinute() / minutes) * minutes);
|
||||
truncated = zdt.withMinute(minute).withSecond(0).withNano(0);
|
||||
} else {
|
||||
// For second-level intervals, truncate to start of second
|
||||
int second = (int) ((zdt.getSecond() / seconds) * seconds);
|
||||
truncated = zdt.withSecond(second).withNano(0);
|
||||
}
|
||||
|
||||
return truncated.toInstant();
|
||||
}
|
||||
|
||||
/**
|
||||
* Generates the cache file path for a given request.
|
||||
*
|
||||
* @param symbol the symbol (ticker, product ID, etc.)
|
||||
* @param startDateTime the start date/time (will be truncated)
|
||||
* @param endDateTime the end date/time (will be truncated)
|
||||
* @param interval the interval duration (used for truncation and filename)
|
||||
* @param notes optional notes section to append to filename (can be
|
||||
* null or empty)
|
||||
* @return the cache file path
|
||||
*/
|
||||
protected Path getCacheFilePath(String symbol, Instant startDateTime, Instant endDateTime, Duration interval,
|
||||
String notes) {
|
||||
Instant truncatedStart = truncateTimestampForCache(startDateTime, interval);
|
||||
Instant truncatedEnd = truncateTimestampForCache(endDateTime, interval);
|
||||
|
||||
String sourcePrefix = getSourceName().isEmpty() ? "" : getSourceName() + "-";
|
||||
String notesSuffix = (notes != null && !notes.trim().isEmpty()) ? "_" + notes.trim() : "";
|
||||
// Use Duration.toString() for standardized format (e.g., "PT24H" for 1 day,
|
||||
// "PT1H" for 1 hour)
|
||||
String durationString = interval.toString();
|
||||
String filename = String.format("%s%s-%s-%d-%d%s.json", sourcePrefix,
|
||||
symbol.toUpperCase().replaceAll("[^A-Z0-9-]", "_"), durationString, truncatedStart.getEpochSecond(),
|
||||
truncatedEnd.getEpochSecond(), notesSuffix);
|
||||
|
||||
return Paths.get(responseCacheDir, filename);
|
||||
}
|
||||
|
||||
/**
|
||||
* Checks if a cache file exists and is still valid based on the interval. For
|
||||
* historical data (end date in the past), cache is valid indefinitely. For
|
||||
* current data (end date is recent), cache expires after the interval duration.
|
||||
* <p>
|
||||
* Cache validity is determined by the file's modification time (when the cache
|
||||
* file was created), not access time. This is appropriate because we want to
|
||||
* know how old the cached data is, not when it was last read.
|
||||
*
|
||||
* @param cacheFile the cache file path
|
||||
* @param interval the interval duration
|
||||
* @param endDateTime the end date/time of the request
|
||||
* @return true if cache is valid, false otherwise
|
||||
*/
|
||||
protected boolean isCacheValid(Path cacheFile, Duration interval, Instant endDateTime) {
|
||||
if (!Files.exists(cacheFile)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
try {
|
||||
// Check file modification time (when the cache file was created/written)
|
||||
Instant fileModified = Files.getLastModifiedTime(cacheFile).toInstant();
|
||||
Instant now = Instant.now();
|
||||
|
||||
// If the request is for historical data (end date is in the past), cache is
|
||||
// valid indefinitely
|
||||
if (endDateTime.isBefore(now.minus(interval))) {
|
||||
return true;
|
||||
}
|
||||
|
||||
// For current/recent data, cache expires after the interval duration
|
||||
Duration age = Duration.between(fileModified, now);
|
||||
return age.compareTo(interval) < 0;
|
||||
} catch (IOException e) {
|
||||
LOG.warn("Failed to check cache file validity: {}", e.getMessage());
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Reads a cached response from disk.
|
||||
*
|
||||
* @param cacheFile the cache file path
|
||||
* @return the cached JSON response, or null if read fails
|
||||
*/
|
||||
protected String readFromCache(Path cacheFile) {
|
||||
try {
|
||||
String content = Files.readString(cacheFile, StandardCharsets.UTF_8);
|
||||
LOG.debug("Cache hit: {}", cacheFile.getFileName());
|
||||
return content;
|
||||
} catch (IOException e) {
|
||||
LOG.warn("Failed to read from cache: {}", e.getMessage());
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Writes a response to the cache.
|
||||
*
|
||||
* @param cacheFile the cache file path
|
||||
* @param response the JSON response to cache
|
||||
*/
|
||||
protected void writeToCache(Path cacheFile, String response) {
|
||||
try {
|
||||
// Ensure parent directory exists
|
||||
Path parentDir = cacheFile.getParent();
|
||||
if (parentDir != null && !Files.exists(parentDir)) {
|
||||
Files.createDirectories(parentDir);
|
||||
}
|
||||
Files.writeString(cacheFile, response, StandardCharsets.UTF_8);
|
||||
LOG.debug("Cached response: {}", cacheFile.getFileName());
|
||||
} catch (IOException e) {
|
||||
LOG.warn("Failed to write to cache: {}", e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Deletes all cache files in the cache directory that belong to this data
|
||||
* source. Files are identified by the source name prefix (e.g., "Coinbase-",
|
||||
* "YahooFinance-").
|
||||
*
|
||||
* @return the number of files deleted
|
||||
*/
|
||||
public int deleteAllCacheFiles() {
|
||||
return deleteCacheFilesOlderThan(Duration.ZERO);
|
||||
}
|
||||
|
||||
/**
|
||||
* Deletes cache files that are older than the specified maximum age. Files are
|
||||
* identified by the source name prefix (e.g., "Coinbase-", "YahooFinance-") and
|
||||
* are deleted based on their file modification time (when the cache file was
|
||||
* created/written).
|
||||
* <p>
|
||||
* <strong>Note:</strong> This method uses file modification time, not access
|
||||
* time. Modification time represents when the cache file was created (when the
|
||||
* response was cached), which is appropriate for determining cache age. Access
|
||||
* time (when the file was last read) is not used because it's not reliably
|
||||
* available across all platforms and filesystems.
|
||||
*
|
||||
* @param maxAge the maximum age of files to keep (files older than this will be
|
||||
* deleted). Use {@link Duration#ZERO} to delete all files.
|
||||
* @return the number of files deleted
|
||||
*/
|
||||
public int deleteCacheFilesOlderThan(Duration maxAge) {
|
||||
if (!Files.exists(Paths.get(responseCacheDir))) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
String sourcePrefix = getSourceName().isEmpty() ? "" : getSourceName() + "-";
|
||||
int deletedCount = 0;
|
||||
Instant cutoffTime = Instant.now().minus(maxAge);
|
||||
|
||||
try {
|
||||
Path cacheDir = Paths.get(responseCacheDir);
|
||||
for (Path path : Files.list(cacheDir).toList()) {
|
||||
if (Files.isRegularFile(path)) {
|
||||
String filename = path.getFileName().toString();
|
||||
// Check if file belongs to this data source
|
||||
if (sourcePrefix.isEmpty() || filename.startsWith(sourcePrefix)) {
|
||||
try {
|
||||
// If maxAge is ZERO, delete all files regardless of age
|
||||
if (maxAge.isZero() || Files.getLastModifiedTime(path).toInstant().isBefore(cutoffTime)) {
|
||||
Files.delete(path);
|
||||
deletedCount++;
|
||||
LOG.debug("Deleted cache file: {}", filename);
|
||||
}
|
||||
} catch (IOException e) {
|
||||
LOG.warn("Failed to delete cache file {}: {}", filename, e.getMessage());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (IOException e) {
|
||||
LOG.warn("Failed to list cache directory: {}", e.getMessage());
|
||||
return deletedCount;
|
||||
}
|
||||
|
||||
if (deletedCount > 0) {
|
||||
LOG.info("Deleted {} stale cache file(s) with prefix '{}'", deletedCount, sourcePrefix);
|
||||
}
|
||||
|
||||
return deletedCount;
|
||||
}
|
||||
|
||||
/**
|
||||
* Deletes stale cache files. A file is considered stale if it's older than 30
|
||||
* days and the end timestamp encoded in the filename indicates it's for
|
||||
* historical data (data that won't change). This is a conservative approach
|
||||
* that preserves recent caches and historical data caches that might still be
|
||||
* useful.
|
||||
* <p>
|
||||
* For more control, use {@link #deleteCacheFilesOlderThan(Duration)} or
|
||||
* {@link #deleteAllCacheFiles()}.
|
||||
*
|
||||
* @return the number of files deleted
|
||||
*/
|
||||
public int deleteStaleCacheFiles() {
|
||||
return deleteStaleCacheFiles(Duration.ofDays(30));
|
||||
}
|
||||
|
||||
/**
|
||||
* Deletes stale cache files older than the specified age. A file is considered
|
||||
* stale if it's older than the specified age. Files are identified by the
|
||||
* source name prefix (e.g., "Coinbase-", "YahooFinance-").
|
||||
*
|
||||
* @param maxAge the maximum age of files to keep (files older than this will be
|
||||
* deleted)
|
||||
* @return the number of files deleted
|
||||
*/
|
||||
public int deleteStaleCacheFiles(Duration maxAge) {
|
||||
return deleteCacheFilesOlderThan(maxAge);
|
||||
}
|
||||
}
|
||||
+45
@@ -0,0 +1,45 @@
|
||||
/*
|
||||
* SPDX-License-Identifier: MIT
|
||||
*/
|
||||
package ta4jexamples.datasources.http;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.net.http.HttpClient;
|
||||
import java.net.http.HttpRequest;
|
||||
import java.net.http.HttpResponse;
|
||||
import java.time.Duration;
|
||||
|
||||
/**
|
||||
* Default implementation of {@link HttpClientWrapper} that delegates to a
|
||||
* {@link HttpClient}.
|
||||
*/
|
||||
public class DefaultHttpClientWrapper implements HttpClientWrapper {
|
||||
|
||||
private final HttpClient httpClient;
|
||||
|
||||
/**
|
||||
* Creates a new wrapper around the specified HttpClient.
|
||||
*
|
||||
* @param httpClient the HttpClient to wrap
|
||||
*/
|
||||
public DefaultHttpClientWrapper(HttpClient httpClient) {
|
||||
if (httpClient == null) {
|
||||
throw new IllegalArgumentException("HttpClient cannot be null");
|
||||
}
|
||||
this.httpClient = httpClient;
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a new wrapper with a default HttpClient.
|
||||
*/
|
||||
public DefaultHttpClientWrapper() {
|
||||
this(HttpClient.newBuilder().connectTimeout(Duration.ofSeconds(10)).build());
|
||||
}
|
||||
|
||||
@Override
|
||||
public <T> HttpResponseWrapper<T> send(HttpRequest request, HttpResponse.BodyHandler<T> handler)
|
||||
throws IOException, InterruptedException {
|
||||
HttpResponse<T> response = httpClient.send(request, handler);
|
||||
return new DefaultHttpResponseWrapper<>(response);
|
||||
}
|
||||
}
|
||||
+39
@@ -0,0 +1,39 @@
|
||||
/*
|
||||
* SPDX-License-Identifier: MIT
|
||||
*/
|
||||
package ta4jexamples.datasources.http;
|
||||
|
||||
import java.net.http.HttpResponse;
|
||||
|
||||
/**
|
||||
* Default implementation of {@link HttpResponseWrapper} that delegates to a
|
||||
* {@link HttpResponse}.
|
||||
*
|
||||
* @param <T> the response body type
|
||||
*/
|
||||
public class DefaultHttpResponseWrapper<T> implements HttpResponseWrapper<T> {
|
||||
|
||||
private final HttpResponse<T> httpResponse;
|
||||
|
||||
/**
|
||||
* Creates a new wrapper around the specified HttpResponse.
|
||||
*
|
||||
* @param httpResponse the HttpResponse to wrap
|
||||
*/
|
||||
public DefaultHttpResponseWrapper(HttpResponse<T> httpResponse) {
|
||||
if (httpResponse == null) {
|
||||
throw new IllegalArgumentException("HttpResponse cannot be null");
|
||||
}
|
||||
this.httpResponse = httpResponse;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int statusCode() {
|
||||
return httpResponse.statusCode();
|
||||
}
|
||||
|
||||
@Override
|
||||
public T body() {
|
||||
return httpResponse.body();
|
||||
}
|
||||
}
|
||||
+46
@@ -0,0 +1,46 @@
|
||||
/*
|
||||
* SPDX-License-Identifier: MIT
|
||||
*/
|
||||
package ta4jexamples.datasources.http;
|
||||
|
||||
import ta4jexamples.datasources.BarSeriesDataSource;
|
||||
|
||||
/**
|
||||
* Marker interface for HTTP-based BarSeries data sources.
|
||||
* <p>
|
||||
* This interface extends {@link BarSeriesDataSource} to identify data sources
|
||||
* that fetch data over HTTP. Implementations of this interface typically use
|
||||
* {@link HttpClientWrapper} for making HTTP requests and may support response
|
||||
* caching.
|
||||
* <p>
|
||||
* HTTP-based data sources share common functionality such as:
|
||||
* <ul>
|
||||
* <li>HTTP request handling via {@link HttpClientWrapper}</li>
|
||||
* <li>Response caching to disk for faster subsequent requests</li>
|
||||
* <li>Cache validation and management</li>
|
||||
* <li>Pagination for large data requests</li>
|
||||
* </ul>
|
||||
* <p>
|
||||
* For shared implementation, consider extending
|
||||
* {@link AbstractHttpBarSeriesDataSource} which provides common HTTP and
|
||||
* caching infrastructure.
|
||||
*
|
||||
* @since 0.20
|
||||
*/
|
||||
public interface HttpBarSeriesDataSource extends BarSeriesDataSource {
|
||||
|
||||
/**
|
||||
* Returns the HttpClientWrapper used by this data source for making HTTP
|
||||
* requests.
|
||||
*
|
||||
* @return the HttpClientWrapper instance
|
||||
*/
|
||||
HttpClientWrapper getHttpClient();
|
||||
|
||||
/**
|
||||
* Returns whether response caching is enabled for this data source.
|
||||
*
|
||||
* @return true if caching is enabled, false otherwise
|
||||
*/
|
||||
boolean isResponseCachingEnabled();
|
||||
}
|
||||
+32
@@ -0,0 +1,32 @@
|
||||
/*
|
||||
* SPDX-License-Identifier: MIT
|
||||
*/
|
||||
package ta4jexamples.datasources.http;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.net.http.HttpRequest;
|
||||
|
||||
/**
|
||||
* Wrapper around {@link java.net.http.HttpClient} to enable easier testing via
|
||||
* mocking.
|
||||
* <p>
|
||||
* This wrapper provides a simple interface for HTTP operations that can be
|
||||
* easily mocked in unit tests, avoiding the need to mock the final
|
||||
* {@link java.net.http.HttpClient} class directly.
|
||||
*/
|
||||
public interface HttpClientWrapper {
|
||||
|
||||
/**
|
||||
* Sends the given request using this client, blocking if necessary to get the
|
||||
* response.
|
||||
*
|
||||
* @param <T> the response body type
|
||||
* @param request the request
|
||||
* @param handler the response body handler
|
||||
* @return the response wrapper
|
||||
* @throws IOException if an I/O error occurs when sending or receiving
|
||||
* @throws InterruptedException if the operation is interrupted
|
||||
*/
|
||||
<T> HttpResponseWrapper<T> send(HttpRequest request, java.net.http.HttpResponse.BodyHandler<T> handler)
|
||||
throws IOException, InterruptedException;
|
||||
}
|
||||
+31
@@ -0,0 +1,31 @@
|
||||
/*
|
||||
* SPDX-License-Identifier: MIT
|
||||
*/
|
||||
package ta4jexamples.datasources.http;
|
||||
|
||||
/**
|
||||
* Wrapper around {@link java.net.http.HttpResponse} to enable easier testing
|
||||
* via mocking.
|
||||
* <p>
|
||||
* This wrapper provides a simple interface for HTTP response operations that
|
||||
* can be easily mocked in unit tests, avoiding the need to mock the final
|
||||
* {@link java.net.http.HttpResponse} class directly.
|
||||
*
|
||||
* @param <T> the response body type
|
||||
*/
|
||||
public interface HttpResponseWrapper<T> {
|
||||
|
||||
/**
|
||||
* Returns the status code for this response.
|
||||
*
|
||||
* @return the status code
|
||||
*/
|
||||
int statusCode();
|
||||
|
||||
/**
|
||||
* Returns the body of this response.
|
||||
*
|
||||
* @return the response body
|
||||
*/
|
||||
T body();
|
||||
}
|
||||
+316
@@ -0,0 +1,316 @@
|
||||
/*
|
||||
* SPDX-License-Identifier: MIT
|
||||
*/
|
||||
package ta4jexamples.datasources.json;
|
||||
|
||||
import com.google.gson.*;
|
||||
import com.google.gson.stream.JsonReader;
|
||||
import com.google.gson.stream.JsonWriter;
|
||||
import org.apache.logging.log4j.LogManager;
|
||||
import org.apache.logging.log4j.Logger;
|
||||
import org.ta4j.core.BarSeries;
|
||||
import org.ta4j.core.BaseBarSeries;
|
||||
import org.ta4j.core.BaseBarSeriesBuilder;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.time.Duration;
|
||||
import java.time.Instant;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Comparator;
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* A TypeAdapter implementation for deserializing JSON data into BarSeries
|
||||
* objects. This adapter supports multiple JSON formats by detecting the
|
||||
* structure of the input data. It currently handles two formats:
|
||||
* <ul>
|
||||
* <li><strong>Coinbase format:</strong> identified by the presence of a
|
||||
* "candles" array. The "start" field represents the start of the candle period,
|
||||
* and the end time is calculated as start + duration. This adapter can be used
|
||||
* for both Coinbase API responses (where the interval is known) and Coinbase
|
||||
* JSON files (where the interval can be inferred from the data).</li>
|
||||
* <li><strong>Binance format:</strong> identified by the presence of an "ohlc"
|
||||
* array</li>
|
||||
* </ul>
|
||||
* <p>
|
||||
* The adapter parses the JSON input and converts it into a BaseBarSeries
|
||||
* instance populated with the appropriate bar data. The bar data is sorted by
|
||||
* timestamp for the Coinbase format to ensure chronological order.
|
||||
* <p>
|
||||
* For Coinbase format, the adapter provides a static helper method
|
||||
* {@link #parseCoinbaseFormat(JsonObject, String, Duration)} that can be used
|
||||
* directly when parsing Coinbase data, allowing you to specify a known interval
|
||||
* (for API responses) or let it be inferred (for JSON files).
|
||||
* <p>
|
||||
* Write operations are not supported by this adapter and will throw an
|
||||
* exception.
|
||||
* <p>
|
||||
* The adapter uses internal lightweight data classes to temporarily hold the
|
||||
* parsed JSON data before converting it into the final BarSeries format.
|
||||
* <p>
|
||||
* Logging is implemented to track which format is being parsed during
|
||||
* deserialization.
|
||||
*
|
||||
* @since 0.19
|
||||
*/
|
||||
public class AdaptiveBarSeriesTypeAdapter extends TypeAdapter<BarSeries> {
|
||||
|
||||
private static final Logger LOG = LogManager.getLogger(AdaptiveBarSeriesTypeAdapter.class);
|
||||
|
||||
@Override
|
||||
public void write(JsonWriter out, BarSeries value) throws IOException {
|
||||
// Not implemented for this use case
|
||||
throw new UnsupportedOperationException("Write operation not supported");
|
||||
}
|
||||
|
||||
/**
|
||||
* Reads a BarSeries from the provided JsonReader by detecting the format based
|
||||
* on available fields. The method parses the JSON input and delegates to
|
||||
* appropriate parsing methods depending on whether the root object contains
|
||||
* "candles" or "ohlc" fields.
|
||||
*
|
||||
* @param in the JsonReader to read the JSON data from
|
||||
* @return the parsed BarSeries object
|
||||
* @throws JsonParseException if the JSON format is unknown or neither "candles"
|
||||
* nor "ohlc" fields are found
|
||||
* @since 0.19
|
||||
*/
|
||||
@Override
|
||||
public BarSeries read(JsonReader in) {
|
||||
JsonElement json = JsonParser.parseReader(in);
|
||||
JsonObject root = json.getAsJsonObject();
|
||||
|
||||
// Detect format based on available fields
|
||||
if (root.has("candles")) {
|
||||
return parseCoinbaseFormat(root);
|
||||
} else if (root.has("ohlc")) {
|
||||
return parseBinanceFormat(root);
|
||||
} else {
|
||||
throw new JsonParseException("Unknown format - neither 'candles' nor 'ohlc' found");
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Parses a JSON object in Coinbase format into a BarSeries. The input JSON is
|
||||
* expected to contain a "candles" array where each element represents a bar
|
||||
* with start time, open, high, low, close, and volume values.
|
||||
* <p>
|
||||
* Note: Coinbase API's "start" field represents the start of the time interval
|
||||
* for the candle. The end time is calculated as start + duration. This method
|
||||
* infers the duration from the data. For cases where the interval is known, use
|
||||
* {@link #parseCoinbaseFormat(JsonObject, String, Duration)} instead.
|
||||
*
|
||||
* @param root the JsonObject representing the root of the JSON data in Coinbase
|
||||
* format
|
||||
* @return a BarSeries populated with data parsed from the Coinbase format JSON
|
||||
*/
|
||||
private BarSeries parseCoinbaseFormat(JsonObject root) {
|
||||
return parseCoinbaseFormat(root, "CoinbaseData", null);
|
||||
}
|
||||
|
||||
/**
|
||||
* Parses a JSON object in Coinbase format into a BarSeries with a known
|
||||
* interval and series name. The input JSON is expected to contain a "candles"
|
||||
* array where each element represents a bar with start time, open, high, low,
|
||||
* close, and volume values.
|
||||
* <p>
|
||||
* Note: Coinbase API's "start" field represents the start of the time interval
|
||||
* for the candle. The end time is calculated as start + duration. This method
|
||||
* uses the provided interval directly instead of inferring it from the data.
|
||||
* <p>
|
||||
* This method can be used for both API responses (where the interval is known)
|
||||
* and JSON files (where the interval can be inferred or provided).
|
||||
*
|
||||
* @param root the JsonObject representing the root of the JSON data in
|
||||
* Coinbase format
|
||||
* @param seriesName the name to use for the BarSeries
|
||||
* @param knownInterval the known bar interval (if null, will be inferred from
|
||||
* the data by calculating the difference between
|
||||
* consecutive start times)
|
||||
* @return a BarSeries populated with data parsed from the Coinbase format JSON
|
||||
*/
|
||||
public static BarSeries parseCoinbaseFormat(JsonObject root, String seriesName, Duration knownInterval) {
|
||||
LOG.trace("Parsing Coinbase format");
|
||||
|
||||
JsonArray candles = root.getAsJsonArray("candles");
|
||||
if (candles == null || candles.isEmpty()) {
|
||||
return new BaseBarSeriesBuilder().withName(seriesName).build();
|
||||
}
|
||||
|
||||
List<CoinbaseBar> barList = new ArrayList<>();
|
||||
|
||||
for (JsonElement candle : candles) {
|
||||
JsonObject candleObj = candle.getAsJsonObject();
|
||||
|
||||
// Skip candles with null or missing required fields
|
||||
if (candleObj.get("start") == null || candleObj.get("start").isJsonNull() || candleObj.get("open") == null
|
||||
|| candleObj.get("open").isJsonNull() || candleObj.get("high") == null
|
||||
|| candleObj.get("high").isJsonNull() || candleObj.get("low") == null
|
||||
|| candleObj.get("low").isJsonNull() || candleObj.get("close") == null
|
||||
|| candleObj.get("close").isJsonNull()) {
|
||||
continue;
|
||||
}
|
||||
|
||||
// Handle null volume by defaulting to "0"
|
||||
String volume = "0";
|
||||
if (candleObj.get("volume") != null && !candleObj.get("volume").isJsonNull()) {
|
||||
volume = candleObj.get("volume").getAsString();
|
||||
}
|
||||
|
||||
// Validate timestamp format before creating CoinbaseBar
|
||||
String startStr = candleObj.get("start").getAsString();
|
||||
try {
|
||||
Long.parseLong(startStr);
|
||||
} catch (NumberFormatException nfe) {
|
||||
LOG.warn("Invalid timestamp format in Coinbase data, skipping candle: {}", startStr, nfe);
|
||||
continue;
|
||||
}
|
||||
|
||||
barList.add(
|
||||
new CoinbaseBar(startStr, candleObj.get("open").getAsString(), candleObj.get("high").getAsString(),
|
||||
candleObj.get("low").getAsString(), candleObj.get("close").getAsString(), volume));
|
||||
}
|
||||
|
||||
// Sort by timestamp (ascending order)
|
||||
barList.sort(Comparator.comparingLong(CoinbaseBar::getStartTime));
|
||||
|
||||
// Build series
|
||||
BaseBarSeries series = new BaseBarSeriesBuilder().withName(seriesName).build();
|
||||
Duration lastDuration = knownInterval;
|
||||
for (int i = 0; i < barList.size(); i++) {
|
||||
CoinbaseBar bar = barList.get(i);
|
||||
// Coinbase "start" field is the start of the candle period
|
||||
Instant startTime = bar.getStartInstant();
|
||||
Duration duration;
|
||||
if (knownInterval != null) {
|
||||
duration = knownInterval;
|
||||
} else {
|
||||
// Infer duration from data by calculating the difference between consecutive
|
||||
// start times
|
||||
Instant previousStart = i > 0 ? barList.get(i - 1).getStartInstant() : null;
|
||||
Instant currentStart = bar.getStartInstant();
|
||||
Instant nextStart = i + 1 < barList.size() ? barList.get(i + 1).getStartInstant() : null;
|
||||
duration = inferDuration(previousStart, currentStart, nextStart, lastDuration);
|
||||
lastDuration = duration;
|
||||
}
|
||||
// End time is start time + duration
|
||||
Instant endTime = startTime.plus(duration);
|
||||
bar.addToSeries(series, endTime, duration);
|
||||
}
|
||||
|
||||
return series;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parses a JSON object in Binance format into a BarSeries. The input JSON is
|
||||
* expected to contain an "ohlc" array where each element represents a bar with
|
||||
* end time, open price, high price, low price, close price, volume, and amount
|
||||
* values.
|
||||
*
|
||||
* @param root the JsonObject representing the root of the JSON data in Binance
|
||||
* format
|
||||
* @return a BarSeries populated with data parsed from the Binance format JSON
|
||||
*/
|
||||
private BarSeries parseBinanceFormat(JsonObject root) {
|
||||
LOG.trace("Parsing Binance format");
|
||||
|
||||
JsonArray ohlc = root.getAsJsonArray("ohlc");
|
||||
String seriesName = root.has("name") ? root.get("name").getAsString() : "BinanceData";
|
||||
|
||||
BaseBarSeries series = new BaseBarSeriesBuilder().withName(seriesName).build();
|
||||
|
||||
List<BinanceBar> bars = new ArrayList<>();
|
||||
for (JsonElement barElement : ohlc) {
|
||||
JsonObject barObj = barElement.getAsJsonObject();
|
||||
BinanceBar bar = new BinanceBar(barObj.get("endTime").getAsLong(), barObj.get("openPrice").getAsNumber(),
|
||||
barObj.get("highPrice").getAsNumber(), barObj.get("lowPrice").getAsNumber(),
|
||||
barObj.get("closePrice").getAsNumber(), barObj.get("volume").getAsNumber(),
|
||||
barObj.get("amount").getAsNumber());
|
||||
bars.add(bar);
|
||||
}
|
||||
|
||||
bars.sort(Comparator.comparingLong(BinanceBar::endTime));
|
||||
|
||||
Duration lastDuration = null;
|
||||
for (int i = 0; i < bars.size(); i++) {
|
||||
BinanceBar bar = bars.get(i);
|
||||
Instant previousEnd = i > 0 ? bars.get(i - 1).getEndInstant() : null;
|
||||
Instant currentEnd = bar.getEndInstant();
|
||||
Instant nextEnd = i + 1 < bars.size() ? bars.get(i + 1).getEndInstant() : null;
|
||||
Duration duration = inferDuration(previousEnd, currentEnd, nextEnd, lastDuration);
|
||||
lastDuration = duration;
|
||||
bar.addToSeries(series, currentEnd, duration);
|
||||
}
|
||||
|
||||
return series;
|
||||
}
|
||||
|
||||
private static Duration inferDuration(Instant previous, Instant current, Instant next, Duration fallback) {
|
||||
Duration candidate = null;
|
||||
if (next != null) {
|
||||
candidate = Duration.between(current, next);
|
||||
} else if (previous != null) {
|
||||
candidate = Duration.between(previous, current);
|
||||
}
|
||||
|
||||
if (candidate != null) {
|
||||
if (candidate.isNegative()) {
|
||||
candidate = candidate.negated();
|
||||
}
|
||||
if (!candidate.isZero()) {
|
||||
return candidate;
|
||||
}
|
||||
}
|
||||
|
||||
if (fallback != null && !fallback.isZero() && !fallback.isNegative()) {
|
||||
return fallback;
|
||||
}
|
||||
|
||||
return Duration.ofSeconds(1);
|
||||
}
|
||||
|
||||
// Lightweight data classes for internal use only
|
||||
private record CoinbaseBar(String start, String open, String high, String low, String close, String volume) {
|
||||
|
||||
public long getStartTime() {
|
||||
return Long.parseLong(start);
|
||||
}
|
||||
|
||||
public Instant getStartInstant() {
|
||||
return Instant.ofEpochSecond(getStartTime());
|
||||
}
|
||||
|
||||
public void addToSeries(BaseBarSeries series, Instant endTime, Duration timePeriod) {
|
||||
series.barBuilder()
|
||||
.timePeriod(timePeriod)
|
||||
.endTime(endTime)
|
||||
.openPrice(open)
|
||||
.highPrice(high)
|
||||
.lowPrice(low)
|
||||
.closePrice(close)
|
||||
.volume(volume)
|
||||
.add();
|
||||
}
|
||||
}
|
||||
|
||||
private record BinanceBar(long endTime, Number openPrice, Number highPrice, Number lowPrice, Number closePrice,
|
||||
Number volume, Number amount) {
|
||||
|
||||
public Instant getEndInstant() {
|
||||
return Instant.ofEpochMilli(endTime);
|
||||
}
|
||||
|
||||
public void addToSeries(BaseBarSeries series, Instant endTimeInstant, Duration timePeriod) {
|
||||
series.barBuilder()
|
||||
.timePeriod(timePeriod)
|
||||
.endTime(endTimeInstant)
|
||||
.openPrice(openPrice)
|
||||
.highPrice(highPrice)
|
||||
.lowPrice(lowPrice)
|
||||
.closePrice(closePrice)
|
||||
.volume(volume)
|
||||
.amount(amount)
|
||||
.add();
|
||||
}
|
||||
}
|
||||
}
|
||||
+59
@@ -0,0 +1,59 @@
|
||||
/*
|
||||
* SPDX-License-Identifier: MIT
|
||||
*/
|
||||
package ta4jexamples.datasources.json;
|
||||
|
||||
import org.ta4j.core.Bar;
|
||||
import org.ta4j.core.BaseBarSeries;
|
||||
import org.ta4j.core.utils.DeprecationNotifier;
|
||||
|
||||
import java.time.Duration;
|
||||
import java.time.Instant;
|
||||
|
||||
@Deprecated(since = "0.19", forRemoval = true)
|
||||
public class GsonBarData {
|
||||
private long endTime;
|
||||
private Number openPrice;
|
||||
private Number highPrice;
|
||||
private Number lowPrice;
|
||||
private Number closePrice;
|
||||
private Number volume;
|
||||
private Number amount;
|
||||
|
||||
public GsonBarData() {
|
||||
DeprecationNotifier.warnOnce(GsonBarData.class, "ta4jexamples.datasources.json.JsonFileBarSeriesDataSource",
|
||||
"0.24.0");
|
||||
}
|
||||
|
||||
@Deprecated(since = "0.19", forRemoval = true)
|
||||
public static GsonBarData from(Bar bar) {
|
||||
DeprecationNotifier.warnOnce(GsonBarData.class, "ta4jexamples.datasources.json.JsonFileBarSeriesDataSource",
|
||||
"0.24.0");
|
||||
var result = new GsonBarData();
|
||||
result.endTime = bar.getEndTime().toEpochMilli();
|
||||
result.openPrice = bar.getOpenPrice().getDelegate();
|
||||
result.highPrice = bar.getHighPrice().getDelegate();
|
||||
result.lowPrice = bar.getLowPrice().getDelegate();
|
||||
result.closePrice = bar.getClosePrice().getDelegate();
|
||||
result.volume = bar.getVolume().getDelegate();
|
||||
result.amount = bar.getAmount().getDelegate();
|
||||
return result;
|
||||
}
|
||||
|
||||
@Deprecated(since = "0.19", forRemoval = true)
|
||||
public void addTo(BaseBarSeries barSeries) {
|
||||
DeprecationNotifier.warnOnce(GsonBarData.class, "ta4jexamples.datasources.json.JsonFileBarSeriesDataSource",
|
||||
"0.24.0");
|
||||
var endTimeInstant = Instant.ofEpochMilli(endTime);
|
||||
barSeries.barBuilder()
|
||||
.timePeriod(Duration.ofDays(1))
|
||||
.endTime(endTimeInstant)
|
||||
.openPrice(openPrice)
|
||||
.highPrice(highPrice)
|
||||
.lowPrice(lowPrice)
|
||||
.closePrice(closePrice)
|
||||
.volume(volume)
|
||||
.amount(amount)
|
||||
.add();
|
||||
}
|
||||
}
|
||||
+50
@@ -0,0 +1,50 @@
|
||||
/*
|
||||
* SPDX-License-Identifier: MIT
|
||||
*/
|
||||
package ta4jexamples.datasources.json;
|
||||
|
||||
import org.ta4j.core.Bar;
|
||||
import org.ta4j.core.BarSeries;
|
||||
import org.ta4j.core.BaseBarSeries;
|
||||
import org.ta4j.core.BaseBarSeriesBuilder;
|
||||
import org.ta4j.core.utils.DeprecationNotifier;
|
||||
|
||||
import java.util.LinkedList;
|
||||
import java.util.List;
|
||||
|
||||
@Deprecated(since = "0.19", forRemoval = true)
|
||||
public class GsonBarSeries {
|
||||
|
||||
private String name;
|
||||
private List<GsonBarData> ohlc = new LinkedList<>();
|
||||
|
||||
public GsonBarSeries() {
|
||||
DeprecationNotifier.warnOnce(GsonBarSeries.class, "ta4jexamples.datasources.json.JsonFileBarSeriesDataSource",
|
||||
"0.24.0");
|
||||
}
|
||||
|
||||
@Deprecated(since = "0.19", forRemoval = true)
|
||||
public static GsonBarSeries from(BarSeries series) {
|
||||
DeprecationNotifier.warnOnce(GsonBarSeries.class, "ta4jexamples.datasources.json.JsonFileBarSeriesDataSource",
|
||||
"0.24.0");
|
||||
GsonBarSeries result = new GsonBarSeries();
|
||||
result.name = series.getName();
|
||||
List<Bar> barData = series.getBarData();
|
||||
for (Bar bar : barData) {
|
||||
GsonBarData exportableBarData = GsonBarData.from(bar);
|
||||
result.ohlc.add(exportableBarData);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
@Deprecated(since = "0.19", forRemoval = true)
|
||||
public BarSeries toBarSeries() {
|
||||
DeprecationNotifier.warnOnce(GsonBarSeries.class, "ta4jexamples.datasources.json.JsonFileBarSeriesDataSource",
|
||||
"0.24.0");
|
||||
BaseBarSeries result = new BaseBarSeriesBuilder().withName(this.name).build();
|
||||
for (GsonBarData data : ohlc) {
|
||||
data.addTo(result);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
}
|
||||
+135
@@ -0,0 +1,135 @@
|
||||
/*
|
||||
* SPDX-License-Identifier: MIT
|
||||
*/
|
||||
package ta4jexamples.datasources.json;
|
||||
|
||||
import com.google.gson.Gson;
|
||||
import com.google.gson.GsonBuilder;
|
||||
import org.apache.logging.log4j.LogManager;
|
||||
import org.apache.logging.log4j.Logger;
|
||||
import org.ta4j.core.BarSeries;
|
||||
import org.ta4j.core.utils.DeprecationNotifier;
|
||||
|
||||
import java.io.*;
|
||||
|
||||
/**
|
||||
* @deprecated
|
||||
* <p>
|
||||
* Use {@link JsonFileBarSeriesDataSource} instead. Scheduled for
|
||||
* removal in 0.24.0.
|
||||
* </p>
|
||||
*/
|
||||
@Deprecated(since = "0.19", forRemoval = true)
|
||||
public class JsonBarsSerializer {
|
||||
|
||||
private static final Logger LOG = LogManager.getLogger(JsonBarsSerializer.class);
|
||||
|
||||
public JsonBarsSerializer() {
|
||||
warnDeprecatedUse();
|
||||
}
|
||||
|
||||
private static void warnDeprecatedUse() {
|
||||
DeprecationNotifier.warnOnce(JsonBarsSerializer.class,
|
||||
"ta4jexamples.datasources.json.JsonFileBarSeriesDataSource", "0.24.0");
|
||||
}
|
||||
|
||||
@Deprecated(since = "0.19", forRemoval = true)
|
||||
public static void persistSeries(BarSeries series, String filename) {
|
||||
warnDeprecatedUse();
|
||||
GsonBarSeries exportableSeries = GsonBarSeries.from(series);
|
||||
Gson gson = new GsonBuilder().setPrettyPrinting().create();
|
||||
FileWriter writer = null;
|
||||
try {
|
||||
writer = new FileWriter(filename);
|
||||
gson.toJson(exportableSeries, writer);
|
||||
LOG.debug("Bar series '{}' successfully saved to '{}'", series.getName(), filename);
|
||||
} catch (IOException e) {
|
||||
LOG.error("Unable to store bars in JSON", e);
|
||||
} finally {
|
||||
if (writer != null) {
|
||||
try {
|
||||
writer.flush();
|
||||
writer.close();
|
||||
} catch (IOException e) {
|
||||
LOG.warn("Unable to flush and/or close file writer", e);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Deprecated(since = "0.19", forRemoval = true)
|
||||
public static BarSeries loadSeries(String filename) {
|
||||
warnDeprecatedUse();
|
||||
Gson gson = new Gson();
|
||||
FileReader reader = null;
|
||||
BarSeries result = null;
|
||||
try {
|
||||
reader = new FileReader(filename);
|
||||
GsonBarSeries loadedSeries = gson.fromJson(reader, GsonBarSeries.class);
|
||||
|
||||
result = loadedSeries.toBarSeries();
|
||||
LOG.debug("Bar series '" + result.getName() + "' successfully loaded. #Entries: " + result.getBarCount());
|
||||
} catch (FileNotFoundException e) {
|
||||
LOG.error("Unable to load bars from JSON", e);
|
||||
} finally {
|
||||
try {
|
||||
if (reader != null)
|
||||
reader.close();
|
||||
} catch (IOException e) {
|
||||
LOG.warn("Unable to close file reader", e);
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Loads a BarSeries from the provided InputStream containing JSON data. The
|
||||
* method parses the JSON content using Gson library and converts it to a
|
||||
* BarSeries object. If the input stream is null or parsing fails, appropriate
|
||||
* warning or error messages are logged and null is returned.
|
||||
*
|
||||
* @deprecated
|
||||
* <p>
|
||||
* Use {@link JsonFileBarSeriesDataSource#loadSeries(String)}
|
||||
* instead.
|
||||
*
|
||||
* @param inputStream the input stream containing JSON data to be parsed into a
|
||||
* BarSeries
|
||||
* @return the loaded BarSeries object, or null if loading fails or input stream
|
||||
* is null
|
||||
*/
|
||||
@Deprecated(since = "0.19", forRemoval = true)
|
||||
public static BarSeries loadSeries(InputStream inputStream) {
|
||||
warnDeprecatedUse();
|
||||
if (inputStream == null) {
|
||||
LOG.warn("Input stream is null, returning null");
|
||||
return null;
|
||||
}
|
||||
|
||||
Gson gson = new Gson();
|
||||
InputStreamReader reader = null;
|
||||
BarSeries result = null;
|
||||
try {
|
||||
reader = new InputStreamReader(inputStream);
|
||||
GsonBarSeries loadedSeries = gson.fromJson(reader, GsonBarSeries.class);
|
||||
|
||||
if (loadedSeries == null) {
|
||||
LOG.warn("Failed to parse JSON, loadedSeries is null");
|
||||
return null;
|
||||
}
|
||||
|
||||
result = loadedSeries.toBarSeries();
|
||||
LOG.debug("Bar series '" + result.getName() + "' successfully loaded. #Entries: " + result.getBarCount());
|
||||
} catch (Exception e) {
|
||||
LOG.error("Unable to load bars from JSON", e);
|
||||
} finally {
|
||||
try {
|
||||
if (reader != null)
|
||||
reader.close();
|
||||
} catch (IOException e) {
|
||||
LOG.warn("Error closing input stream reader", e);
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user